Add PolarisMetaStoreManager.loadEntities

persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java

@@ -428,7 +428,7 @@ public List<EntityNameLookupRecord> lookupEntityActiveBatchInCurrentTxn(
   }
 
   @Override
-  public @Nonnull <T> Page<T> listEntitiesInCurrentTxn(
+  public @Nonnull <T> Page<T> loadEntitiesInCurrentTxn(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java

@@ -431,7 +431,7 @@ public Page<EntityNameLookupRecord> listEntities(
       @Nonnull PolarisEntitySubType entitySubType,
       @Nonnull PageToken pageToken) {
     // TODO: only fetch the properties required for creating an EntityNameLookupRecord
-    return listEntities(
+    return loadEntities(
         callCtx,
         catalogId,
         parentId,
@@ -444,7 +444,7 @@ public Page<EntityNameLookupRecord> listEntities(
 
   @Nonnull
   @Override
-  public <T> Page<T> listEntities(
+  public <T> Page<T> loadEntities(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,

polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java

@@ -223,11 +223,6 @@ public PolarisEntitySubType getSubType() {
     return PolarisEntitySubType.fromCode(getSubTypeCode());
   }
 
-  @JsonIgnore
-  public NameAndId nameAndId() {
-    return new NameAndId(name, id);
-  }
-
   @Override
   public String toString() {
     return "name="

polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java

@@ -717,10 +717,45 @@ private void revokeGrantRecord(
     // with sensitive data; but the window of inconsistency is only the duration of a single
     // in-flight request (the cache-based resolution follows a different path entirely).
 
-    // done
     return ListEntitiesResult.fromPage(resultPage);
   }
 
+  /** {@inheritDoc} */
+  @Override
+  public @Nonnull Page<PolarisBaseEntity> loadEntities(
+      @Nonnull PolarisCallContext callCtx,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType,
+      @Nonnull PageToken pageToken) {
+    // get meta store we should be using
+    BasePersistence ms = callCtx.getMetaStore();
+
+    // return list of active entities
+    // TODO: Clean up shared logic for catalogId/parentId
+    long catalogId = catalogPath == null || catalogPath.isEmpty() ? 0L : catalogPath.get(0).getId();
+    long parentId =
+        catalogPath == null || catalogPath.isEmpty()
+            ? 0L
+            : catalogPath.get(catalogPath.size() - 1).getId();
+
+    // TODO: Use post-validation to enforce consistent view against catalogPath. In the
+    // meantime, happens-before ordering semantics aren't guaranteed during high-concurrency
+    // race conditions, such as first revoking a grant on a namespace before adding a table
+    // with sensitive data; but the window of inconsistency is only the duration of a single
+    // in-flight request (the cache-based resolution follows a different path entirely).
+
+    return ms.loadEntities(
+        callCtx,
+        catalogId,
+        parentId,
+        entityType,
+        entitySubType,
+        entity -> true,
+        Function.identity(),
+        pageToken);
+  }
+
   /** {@inheritDoc} */
   @Override
   public @Nonnull CreatePrincipalResult createPrincipal(
@@ -1166,7 +1201,7 @@ private void revokeGrantRecord(
 
       // get the list of catalog roles, at most 2
       List<PolarisBaseEntity> catalogRoles =
-          ms.listEntities(
+          ms.loadEntities(
                   callCtx,
                   catalogId,
                   catalogId,
@@ -1488,7 +1523,7 @@ private void revokeGrantRecord(
 
     // find all available tasks
     Page<PolarisBaseEntity> availableTasks =
-        ms.listEntities(
+        ms.loadEntities(
             callCtx,
             PolarisEntityConstants.getRootEntityId(),
             PolarisEntityConstants.getRootEntityId(),

polaris-core/src/main/java/org/apache/polaris/core/persistence/BasePersistence.java

@@ -270,15 +270,16 @@ List<PolarisChangeTrackingVersions> lookupEntityVersions(
       @Nonnull PolarisCallContext callCtx, List<PolarisEntityId> entityIds);
 
   /**
-   * List all entities of the specified type which are child entities of the specified parent
+   * List lightweight information of entities matching the given criteria with pagination. If all
+   * properties of the entity are required,use {@link #loadEntities} instead.
    *
    * @param callCtx call context
    * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level
    * @param parentId id of the parent, can be the special 0 value representing the root entity
    * @param entityType type of entities to list
    * @param entitySubType subType of entities to list (or ANY_SUBTYPE)
    * @param pageToken the token to start listing after
-   * @return the list of entities for the specified list operation
+   * @return the paged list of matching entities
    */
   @Nonnull
   Page<EntityNameLookupRecord> listEntities(
@@ -290,7 +291,8 @@ Page<EntityNameLookupRecord> listEntities(
       @Nonnull PageToken pageToken);
 
   /**
-   * List entities where some predicate returns true and transform the entities with a function
+   * Load full entities matching the given criteria with pagination and transformation. If only the
+   * entity name/id/type is required, use {@link #listEntities} instead.
    *
    * @param callCtx call context
    * @param catalogId catalog id for that entity, NULL_ID if the entity is top-level
@@ -301,10 +303,10 @@ Page<EntityNameLookupRecord> listEntities(
    *     returns true are returned in the list
    * @param transformer the transformation function applied to the {@link PolarisBaseEntity} before
    *     returning
-   * @return the list of entities for which the predicate returns true
+   * @return the paged list of matching entities after transformation
    */
   @Nonnull
-  <T> Page<T> listEntities(
+  <T> Page<T> loadEntities(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java

@@ -47,6 +47,7 @@
 import org.apache.polaris.core.persistence.dao.entity.GenerateEntityIdResult;
 import org.apache.polaris.core.persistence.dao.entity.ListEntitiesResult;
 import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult;
+import org.apache.polaris.core.persistence.pagination.Page;
 import org.apache.polaris.core.persistence.pagination.PageToken;
 import org.apache.polaris.core.policy.PolarisPolicyMappingManager;
 import org.apache.polaris.core.storage.PolarisCredentialVendor;
@@ -110,8 +111,8 @@ EntityResult readEntityByName(
       @Nonnull String name);
 
   /**
-   * List all entities of the specified type under the specified catalogPath. If the catalogPath is
-   * null, listed entities will be top-level entities like catalogs.
+   * List lightweight information about entities matching the given criteria. If all properties of
+   * the entity are required,use {@link #loadEntities} instead.
    *
    * @param callCtx call context
    * @param catalogPath path inside a catalog. If null or empty, the entities to list are top-level,
@@ -129,6 +130,47 @@ ListEntitiesResult listEntities(
       @Nonnull PolarisEntitySubType entitySubType,
       @Nonnull PageToken pageToken);
 
+  /**
+   * Load full entities matching the given criteria with pagination. If only the entity name/id/type
+   * is required, use {@link #listEntities} instead. If no pagination is required, use {@link
+   * #loadEntitiesAll} instead.
+   *
+   * @param callCtx call context
+   * @param catalogPath path inside a catalog. If null or empty, the entities to list are top-level,
+   *     like catalogs
+   * @param entityType type of entities to list
+   * @param entitySubType subType of entities to list (or ANY_SUBTYPE)
+   * @return paged list of matching entities
+   */
+  @Nonnull
+  Page<PolarisBaseEntity> loadEntities(
+      @Nonnull PolarisCallContext callCtx,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType,
+      @Nonnull PageToken pageToken);
+
+  /**
+   * Load full entities matching the given criteria into an unpaged list. If pagination is required
+   * use {@link #loadEntities} instead. If only the entity name/id/type is required, use {@link
+   * #listEntities} instead.
+   *
+   * @param callCtx call context
+   * @param catalogPath path inside a catalog. If null or empty, the entities to list are top-level,
+   *     like catalogs
+   * @param entityType type of entities to list
+   * @param entitySubType subType of entities to list (or ANY_SUBTYPE)
+   * @return list of all matching entities
+   */
+  default @Nonnull List<PolarisBaseEntity> loadEntitiesAll(
+      @Nonnull PolarisCallContext callCtx,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType) {
+    return loadEntities(callCtx, catalogPath, entityType, entitySubType, PageToken.readEverything())
+        .items();
+  }
+
   /**
    * Generate a new unique id that can be used by the Polaris client when it needs to create a new
    * entity

polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java

@@ -53,6 +53,7 @@
 import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult;
 import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult;
 import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult;
+import org.apache.polaris.core.persistence.pagination.Page;
 import org.apache.polaris.core.persistence.pagination.PageToken;
 import org.apache.polaris.core.policy.PolicyEntity;
 import org.apache.polaris.core.policy.PolicyType;
@@ -119,7 +120,7 @@ public EntityResult readEntityByName(
   }
 
   @Override
-  public ListEntitiesResult listEntities(
+  public @Nonnull ListEntitiesResult listEntities(
       @Nonnull PolarisCallContext callCtx,
       @Nullable List<PolarisEntityCore> catalogPath,
       @Nonnull PolarisEntityType entityType,
@@ -129,6 +130,17 @@ public ListEntitiesResult listEntities(
     return null;
   }
 
+  @Override
+  public @Nonnull Page<PolarisBaseEntity> loadEntities(
+      @Nonnull PolarisCallContext callCtx,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType,
+      @Nonnull PageToken pageToken) {
+    callCtx.getDiagServices().fail("illegal_method_in_transaction_workspace", "loadEntities");
+    return null;
+  }
+
   @Override
   public GenerateEntityIdResult generateNewEntityId(@Nonnull PolarisCallContext callCtx) {
     diagnostics.fail("illegal_method_in_transaction_workspace", "generateNewEntityId");

polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/AbstractTransactionalPersistence.java

@@ -383,7 +383,7 @@ public Page<EntityNameLookupRecord> listEntities(
   /** {@inheritDoc} */
   @Override
   @Nonnull
-  public <T> Page<T> listEntities(
+  public <T> Page<T> loadEntities(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,
@@ -395,7 +395,7 @@ public <T> Page<T> listEntities(
     return runInReadTransaction(
         callCtx,
         () ->
-            this.listEntitiesInCurrentTxn(
+            this.loadEntitiesInCurrentTxn(
                 callCtx,
                 catalogId,
                 parentId,

polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalMetaStoreManagerImpl.java

@@ -699,7 +699,6 @@ private void bootstrapPolarisService(
             entitySubType,
             pageToken);
 
-    // done
     return ListEntitiesResult.fromPage(resultPage);
   }
 
@@ -720,6 +719,54 @@ private void bootstrapPolarisService(
         () -> listEntities(callCtx, ms, catalogPath, entityType, entitySubType, pageToken));
   }
 
+  /**
+   * See {@link PolarisMetaStoreManager#loadEntities(PolarisCallContext, List, PolarisEntityType,
+   * PolarisEntitySubType, PageToken)}
+   */
+  private @Nonnull Page<PolarisBaseEntity> loadEntities(
+      @Nonnull PolarisCallContext callCtx,
+      @Nonnull TransactionalPersistence ms,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType,
+      @Nonnull PageToken pageToken) {
+    // first resolve again the catalogPath to that entity
+    PolarisEntityResolver resolver = new PolarisEntityResolver(callCtx, ms, catalogPath);
+
+    // throw if we failed to resolve
+    if (resolver.isFailure()) {
+      throw new IllegalArgumentException("Failed to resolve catalogPath: " + catalogPath);
+    }
+
+    // return list of active entities
+    return ms.loadEntitiesInCurrentTxn(
+        callCtx,
+        resolver.getCatalogIdOrNull(),
+        resolver.getParentId(),
+        entityType,
+        entitySubType,
+        entity -> true,
+        Function.identity(),
+        pageToken);
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public @Nonnull Page<PolarisBaseEntity> loadEntities(
+      @Nonnull PolarisCallContext callCtx,
+      @Nullable List<PolarisEntityCore> catalogPath,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull PolarisEntitySubType entitySubType,
+      @Nonnull PageToken pageToken) {
+    // get meta store we should be using
+    TransactionalPersistence ms = ((TransactionalPersistence) callCtx.getMetaStore());
+
+    // run operation in a read transaction
+    return ms.runInReadTransaction(
+        callCtx,
+        () -> loadEntities(callCtx, ms, catalogPath, entityType, entitySubType, pageToken));
+  }
+
   /** {@link #createPrincipal(PolarisCallContext, PolarisBaseEntity)} */
   private @Nonnull CreatePrincipalResult createPrincipal(
       @Nonnull PolarisCallContext callCtx,
@@ -1335,7 +1382,7 @@ private void bootstrapPolarisService(
 
       // get the list of catalog roles, at most 2
       List<PolarisBaseEntity> catalogRoles =
-          ms.listEntitiesInCurrentTxn(
+          ms.loadEntitiesInCurrentTxn(
                   callCtx,
                   catalogId,
                   catalogId,
@@ -1901,7 +1948,7 @@ private PolarisEntityResolver resolveSecurableToRoleGrant(
 
     // find all available tasks
     Page<PolarisBaseEntity> availableTasks =
-        ms.listEntitiesInCurrentTxn(
+        ms.loadEntitiesInCurrentTxn(
             callCtx,
             PolarisEntityConstants.getRootEntityId(),
             PolarisEntityConstants.getRootEntityId(),

polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TransactionalPersistence.java

@@ -214,7 +214,7 @@ List<PolarisChangeTrackingVersions> lookupEntityVersionsInCurrentTxn(
       @Nonnull PolarisEntityType entityType,
       @Nonnull PolarisEntitySubType entitySubType,
       @Nonnull PageToken pageToken) {
-    return listEntitiesInCurrentTxn(
+    return loadEntitiesInCurrentTxn(
         callCtx,
         catalogId,
         parentId,
@@ -225,9 +225,9 @@ List<PolarisChangeTrackingVersions> lookupEntityVersionsInCurrentTxn(
         pageToken);
   }
 
-  /** See {@link org.apache.polaris.core.persistence.BasePersistence#listEntities} */
+  /** See {@link org.apache.polaris.core.persistence.BasePersistence#loadEntities} */
   @Nonnull
-  <T> Page<T> listEntitiesInCurrentTxn(
+  <T> Page<T> loadEntitiesInCurrentTxn(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,

polaris-core/src/main/java/org/apache/polaris/core/persistence/transactional/TreeMapTransactionalPersistenceImpl.java

@@ -288,15 +288,7 @@ public EntityNameLookupRecord lookupEntityActiveInCurrentTxn(
                     entityActiveKey.getName()));
 
     // return record
-    return (entity == null)
-        ? null
-        : new EntityNameLookupRecord(
-            entity.getCatalogId(),
-            entity.getId(),
-            entity.getParentId(),
-            entity.getName(),
-            entity.getTypeCode(),
-            entity.getSubTypeCode());
+    return entity == null ? null : new EntityNameLookupRecord(entity);
   }
 
   /** {@inheritDoc} */
@@ -312,7 +304,7 @@ public List<EntityNameLookupRecord> lookupEntityActiveBatchInCurrentTxn(
   }
 
   @Override
-  public @Nonnull <T> Page<T> listEntitiesInCurrentTxn(
+  public @Nonnull <T> Page<T> loadEntitiesInCurrentTxn(
       @Nonnull PolarisCallContext callCtx,
       long catalogId,
       long parentId,