Remove all tightly coupled EntityCache dependencies in the main persistence stack

polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java

@@ -25,12 +25,23 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
 
+/**
+ * For legacy reasons, this class is only a thin facade over PolarisBaseEntity's members/methods. No
+ * direct members should be added to this class; rather, they should reside in the PolarisBaseEntity
+ * and this class should just contain the relevant builder methods, etc. The intention when using
+ * this class is to use "immutable" semantics as much as possible, for example constructing new
+ * copies with the Builder pattern when "mutating" fields rather than ever chaing fields in-place.
+ * Currently, code that intends to operate directly on a PolarisBaseEntity may not adhere to
+ * immutability semantics, and may modify the entity in-place.
+ *
+ * <p>TODO: Combine this fully into PolarisBaseEntity, refactor all callsites to use strict
+ * immutability semantics, and remove all mutator methods from PolarisBaseEntity.
+ */
 public class PolarisEntity extends PolarisBaseEntity {
 
   public static class NameAndId {
@@ -227,41 +238,18 @@ public String toString() {
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
-    if (!(o instanceof PolarisEntity)) return false;
-    PolarisEntity that = (PolarisEntity) o;
-    return catalogId == that.catalogId
-        && id == that.id
-        && parentId == that.parentId
-        && createTimestamp == that.createTimestamp
-        && dropTimestamp == that.dropTimestamp
-        && purgeTimestamp == that.purgeTimestamp
-        && lastUpdateTimestamp == that.lastUpdateTimestamp
-        && entityVersion == that.entityVersion
-        && grantRecordsVersion == that.grantRecordsVersion
-        && typeCode == that.typeCode
-        && subTypeCode == that.subTypeCode
-        && Objects.equals(name, that.name)
-        && Objects.equals(properties, that.properties)
-        && Objects.equals(internalProperties, that.internalProperties);
+    // Note: Keeping this here explicitly instead silently inheriting super.equals as a more
+    // prominent warning that the data members of this class *must not* diverge from those of
+    // PolarisBaseEntity.
+    return super.equals(o);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(
-        typeCode,
-        subTypeCode,
-        catalogId,
-        id,
-        parentId,
-        name,
-        createTimestamp,
-        dropTimestamp,
-        purgeTimestamp,
-        lastUpdateTimestamp,
-        properties,
-        internalProperties,
-        entityVersion,
-        grantRecordsVersion);
+    // Note: Keeping this here explicitly instead silently inheriting super.hashCode as a more
+    // prominent warning that the data members of this class *must not* diverge from those of
+    // PolarisBaseEntity.
+    return super.hashCode();
   }
 
   public static class Builder extends BaseBuilder<PolarisEntity, Builder> {

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java

@@ -29,24 +29,23 @@
 import org.apache.polaris.core.auth.PolarisGrantManager;
 import org.apache.polaris.core.auth.PolarisSecretsManager;
 import org.apache.polaris.core.entity.PolarisBaseEntity;
+import org.apache.polaris.core.entity.PolarisChangeTrackingVersions;
 import org.apache.polaris.core.entity.PolarisEntity;
 import org.apache.polaris.core.entity.PolarisEntityActiveRecord;
 import org.apache.polaris.core.entity.PolarisEntityCore;
+import org.apache.polaris.core.entity.PolarisEntityId;
 import org.apache.polaris.core.entity.PolarisEntitySubType;
 import org.apache.polaris.core.entity.PolarisEntityType;
+import org.apache.polaris.core.entity.PolarisGrantRecord;
 import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
-import org.apache.polaris.core.persistence.cache.PolarisRemoteCache;
 import org.apache.polaris.core.storage.PolarisCredentialVendor;
 
 /**
  * Polaris Metastore Manager manages all Polaris entities and associated grant records metadata for
  * authorization. It uses the underlying persistent metastore to store and retrieve Polaris metadata
  */
 public interface PolarisMetaStoreManager
-    extends PolarisSecretsManager,
-        PolarisGrantManager,
-        PolarisRemoteCache,
-        PolarisCredentialVendor {
+    extends PolarisSecretsManager, PolarisGrantManager, PolarisCredentialVendor {
 
   /**
    * Bootstrap the Polaris service, creating the root catalog, root principal, and associated
@@ -688,4 +687,198 @@ DropEntityResult dropEntityIfExists(
    */
   @Nonnull
   EntitiesResult loadTasks(@Nonnull PolarisCallContext callCtx, String executorId, int limit);
+
+  /** Result of a loadEntitiesChangeTracking call */
+  class ChangeTrackingResult extends BaseResult {
+
+    // null if not success. Else, will be null if the grant to revoke was not found
+    private final List<PolarisChangeTrackingVersions> changeTrackingVersions;
+
+    /**
+     * Constructor for an error
+     *
+     * @param errorCode error code, cannot be SUCCESS
+     * @param extraInformation extra information
+     */
+    public ChangeTrackingResult(
+        @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) {
+      super(errorCode, extraInformation);
+      this.changeTrackingVersions = null;
+    }
+
+    /**
+     * Constructor for success
+     *
+     * @param changeTrackingVersions change tracking versions
+     */
+    public ChangeTrackingResult(
+        @Nonnull List<PolarisChangeTrackingVersions> changeTrackingVersions) {
+      super(BaseResult.ReturnStatus.SUCCESS);
+      this.changeTrackingVersions = changeTrackingVersions;
+    }
+
+    @JsonCreator
+    private ChangeTrackingResult(
+        @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus,
+        @JsonProperty("extraInformation") String extraInformation,
+        @JsonProperty("changeTrackingVersions")
+            List<PolarisChangeTrackingVersions> changeTrackingVersions) {
+      super(returnStatus, extraInformation);
+      this.changeTrackingVersions = changeTrackingVersions;
+    }
+
+    public List<PolarisChangeTrackingVersions> getChangeTrackingVersions() {
+      return changeTrackingVersions;
+    }
+  }
+
+  /**
+   * Load change tracking information for a set of entities in one single shot and return for each
+   * the version for the entity itself and the version associated to its grant records.
+   *
+   * @param callCtx call context
+   * @param entityIds list of catalog/entity pair ids for which we need to efficiently load the
+   *     version information, both entity version and grant records version.
+   * @return a list of version tracking information. Order in that returned list is the same as the
+   *     input list. Some elements might be NULL if the entity has been purged. Not expected to fail
+   */
+  @Nonnull
+  ChangeTrackingResult loadEntitiesChangeTracking(
+      @Nonnull PolarisCallContext callCtx, @Nonnull List<PolarisEntityId> entityIds);
+
+  /**
+   * Represents an entity with its grants. If we "refresh" a previously fetched entity, we will only
+   * refresh the information which has changed, based on the version of the entity.
+   */
+  class ResolvedEntityResult extends BaseResult {
+
+    // the entity itself if it was loaded
+    private final @Nullable PolarisBaseEntity entity;
+
+    // version for the grant records, in case the entity was not loaded
+    private final int grantRecordsVersion;
+
+    private final @Nullable List<PolarisGrantRecord> entityGrantRecords;
+
+    /**
+     * Constructor for an error
+     *
+     * @param errorCode error code, cannot be SUCCESS
+     * @param extraInformation extra information
+     */
+    public ResolvedEntityResult(
+        @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) {
+      super(errorCode, extraInformation);
+      this.entity = null;
+      this.entityGrantRecords = null;
+      this.grantRecordsVersion = 0;
+    }
+
+    /**
+     * Constructor with success
+     *
+     * @param entity the main entity
+     * @param grantRecordsVersion the version of the grant records
+     * @param entityGrantRecords the list of grant records
+     */
+    public ResolvedEntityResult(
+        @Nullable PolarisBaseEntity entity,
+        int grantRecordsVersion,
+        @Nullable List<PolarisGrantRecord> entityGrantRecords) {
+      super(BaseResult.ReturnStatus.SUCCESS);
+      this.entity = entity;
+      this.entityGrantRecords = entityGrantRecords;
+      this.grantRecordsVersion = grantRecordsVersion;
+    }
+
+    @JsonCreator
+    public ResolvedEntityResult(
+        @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus,
+        @JsonProperty("extraInformation") String extraInformation,
+        @Nullable @JsonProperty("entity") PolarisBaseEntity entity,
+        @JsonProperty("grantRecordsVersion") int grantRecordsVersion,
+        @Nullable @JsonProperty("entityGrantRecords") List<PolarisGrantRecord> entityGrantRecords) {
+      super(returnStatus, extraInformation);
+      this.entity = entity;
+      this.entityGrantRecords = entityGrantRecords;
+      this.grantRecordsVersion = grantRecordsVersion;
+    }
+
+    public @Nullable PolarisBaseEntity getEntity() {
+      return entity;
+    }
+
+    public int getGrantRecordsVersion() {
+      return grantRecordsVersion;
+    }
+
+    public @Nullable List<PolarisGrantRecord> getEntityGrantRecords() {
+      return entityGrantRecords;
+    }
+  }
+
+  /**
+   * Load a resolved entity, i.e. an entity definition and associated grant records, from the
+   * backend store. The entity is identified by its id (entity catalog id and id).
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityCatalogId id of the catalog for that entity
+   * @param entityId id of the entity
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult loadResolvedEntityById(
+      @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId);
+
+  /**
+   * Load a resolved entity, i.e. an entity definition and associated grant records, from the
+   * backend store. The entity is identified by its name. Will return NULL if the entity does not
+   * exist, i.e. has been purged or dropped.
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityCatalogId id of the catalog for that entity
+   * @param parentId the id of the parent of that entity
+   * @param entityType the type of this entity
+   * @param entityName the name of this entity
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult loadResolvedEntityByName(
+      @Nonnull PolarisCallContext callCtx,
+      long entityCatalogId,
+      long parentId,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull String entityName);
+
+  /**
+   * Refresh a resolved entity from the backend store. Will return NULL if the entity does not
+   * exist, i.e. has been purged or dropped. Else, will determine what has changed based on the
+   * version information sent by the caller and will return only what has changed.
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityType type of the entity whose entity and grants we are refreshing
+   * @param entityCatalogId id of the catalog for that entity
+   * @param entityId the id of the entity to load
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult refreshResolvedEntity(
+      @Nonnull PolarisCallContext callCtx,
+      int entityVersion,
+      int entityGrantRecordsVersion,
+      @Nonnull PolarisEntityType entityType,
+      long entityCatalogId,
+      long entityId);
 }