Remove all tightly coupled EntityCache dependencies in the main persistence stack (#1055)

Remove the EntityCacheEntry wrapper since the timestamp fields were never used anyways;
instead the underlying Caffeine cache transparently handles access times, and the
types of entries we cache are simply the existing ResolvedPolarisEntity.

Remove interactions of business logic with explicit "cache entries", instead
operating on ResolvedPolarisEntity. Fix the equals()/hashCode() behavior of
PolarisEntity to be compatible with PolarisBaseEntity as intended.

Improve code comments to explain the (current) relationship between PolarisEntity
and PolarisBaseEntity, and clarify the behaviors in Resolver.java.

Fully remove the PolarisRemoteCache interface and its methods. Add different
methods that aren't cache-specific instead.

Author: dennishuo

polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntity.java

@@ -25,12 +25,23 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
 
+/**
+ * For legacy reasons, this class is only a thin facade over PolarisBaseEntity's members/methods. No
+ * direct members should be added to this class; rather, they should reside in the PolarisBaseEntity
+ * and this class should just contain the relevant builder methods, etc. The intention when using
+ * this class is to use "immutable" semantics as much as possible, for example constructing new
+ * copies with the Builder pattern when "mutating" fields rather than ever chaing fields in-place.
+ * Currently, code that intends to operate directly on a PolarisBaseEntity may not adhere to
+ * immutability semantics, and may modify the entity in-place.
+ *
+ * <p>TODO: Combine this fully into PolarisBaseEntity, refactor all callsites to use strict
+ * immutability semantics, and remove all mutator methods from PolarisBaseEntity.
+ */
 public class PolarisEntity extends PolarisBaseEntity {
 
   public static class NameAndId {
@@ -227,41 +238,18 @@ public String toString() {
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
-    if (!(o instanceof PolarisEntity)) return false;
-    PolarisEntity that = (PolarisEntity) o;
-    return catalogId == that.catalogId
-        && id == that.id
-        && parentId == that.parentId
-        && createTimestamp == that.createTimestamp
-        && dropTimestamp == that.dropTimestamp
-        && purgeTimestamp == that.purgeTimestamp
-        && lastUpdateTimestamp == that.lastUpdateTimestamp
-        && entityVersion == that.entityVersion
-        && grantRecordsVersion == that.grantRecordsVersion
-        && typeCode == that.typeCode
-        && subTypeCode == that.subTypeCode
-        && Objects.equals(name, that.name)
-        && Objects.equals(properties, that.properties)
-        && Objects.equals(internalProperties, that.internalProperties);
+    // Note: Keeping this here explicitly instead silently inheriting super.equals as a more
+    // prominent warning that the data members of this class *must not* diverge from those of
+    // PolarisBaseEntity.
+    return super.equals(o);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(
-        typeCode,
-        subTypeCode,
-        catalogId,
-        id,
-        parentId,
-        name,
-        createTimestamp,
-        dropTimestamp,
-        purgeTimestamp,
-        lastUpdateTimestamp,
-        properties,
-        internalProperties,
-        entityVersion,
-        grantRecordsVersion);
+    // Note: Keeping this here explicitly instead silently inheriting super.hashCode as a more
+    // prominent warning that the data members of this class *must not* diverge from those of
+    // PolarisBaseEntity.
+    return super.hashCode();
   }
 
   public static class Builder extends BaseBuilder<PolarisEntity, Builder> {

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java

@@ -29,24 +29,23 @@
 import org.apache.polaris.core.auth.PolarisGrantManager;
 import org.apache.polaris.core.auth.PolarisSecretsManager;
 import org.apache.polaris.core.entity.PolarisBaseEntity;
+import org.apache.polaris.core.entity.PolarisChangeTrackingVersions;
 import org.apache.polaris.core.entity.PolarisEntity;
 import org.apache.polaris.core.entity.PolarisEntityActiveRecord;
 import org.apache.polaris.core.entity.PolarisEntityCore;
+import org.apache.polaris.core.entity.PolarisEntityId;
 import org.apache.polaris.core.entity.PolarisEntitySubType;
 import org.apache.polaris.core.entity.PolarisEntityType;
+import org.apache.polaris.core.entity.PolarisGrantRecord;
 import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
-import org.apache.polaris.core.persistence.cache.PolarisRemoteCache;
 import org.apache.polaris.core.storage.PolarisCredentialVendor;
 
 /**
  * Polaris Metastore Manager manages all Polaris entities and associated grant records metadata for
  * authorization. It uses the underlying persistent metastore to store and retrieve Polaris metadata
  */
 public interface PolarisMetaStoreManager
-    extends PolarisSecretsManager,
-        PolarisGrantManager,
-        PolarisRemoteCache,
-        PolarisCredentialVendor {
+    extends PolarisSecretsManager, PolarisGrantManager, PolarisCredentialVendor {
 
   /**
    * Bootstrap the Polaris service, creating the root catalog, root principal, and associated
@@ -688,4 +687,198 @@ DropEntityResult dropEntityIfExists(
    */
   @Nonnull
   EntitiesResult loadTasks(@Nonnull PolarisCallContext callCtx, String executorId, int limit);
+
+  /** Result of a loadEntitiesChangeTracking call */
+  class ChangeTrackingResult extends BaseResult {
+
+    // null if not success. Else, will be null if the grant to revoke was not found
+    private final List<PolarisChangeTrackingVersions> changeTrackingVersions;
+
+    /**
+     * Constructor for an error
+     *
+     * @param errorCode error code, cannot be SUCCESS
+     * @param extraInformation extra information
+     */
+    public ChangeTrackingResult(
+        @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) {
+      super(errorCode, extraInformation);
+      this.changeTrackingVersions = null;
+    }
+
+    /**
+     * Constructor for success
+     *
+     * @param changeTrackingVersions change tracking versions
+     */
+    public ChangeTrackingResult(
+        @Nonnull List<PolarisChangeTrackingVersions> changeTrackingVersions) {
+      super(BaseResult.ReturnStatus.SUCCESS);
+      this.changeTrackingVersions = changeTrackingVersions;
+    }
+
+    @JsonCreator
+    private ChangeTrackingResult(
+        @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus,
+        @JsonProperty("extraInformation") String extraInformation,
+        @JsonProperty("changeTrackingVersions")
+            List<PolarisChangeTrackingVersions> changeTrackingVersions) {
+      super(returnStatus, extraInformation);
+      this.changeTrackingVersions = changeTrackingVersions;
+    }
+
+    public List<PolarisChangeTrackingVersions> getChangeTrackingVersions() {
+      return changeTrackingVersions;
+    }
+  }
+
+  /**
+   * Load change tracking information for a set of entities in one single shot and return for each
+   * the version for the entity itself and the version associated to its grant records.
+   *
+   * @param callCtx call context
+   * @param entityIds list of catalog/entity pair ids for which we need to efficiently load the
+   *     version information, both entity version and grant records version.
+   * @return a list of version tracking information. Order in that returned list is the same as the
+   *     input list. Some elements might be NULL if the entity has been purged. Not expected to fail
+   */
+  @Nonnull
+  ChangeTrackingResult loadEntitiesChangeTracking(
+      @Nonnull PolarisCallContext callCtx, @Nonnull List<PolarisEntityId> entityIds);
+
+  /**
+   * Represents an entity with its grants. If we "refresh" a previously fetched entity, we will only
+   * refresh the information which has changed, based on the version of the entity.
+   */
+  class ResolvedEntityResult extends BaseResult {
+
+    // the entity itself if it was loaded
+    private final @Nullable PolarisBaseEntity entity;
+
+    // version for the grant records, in case the entity was not loaded
+    private final int grantRecordsVersion;
+
+    private final @Nullable List<PolarisGrantRecord> entityGrantRecords;
+
+    /**
+     * Constructor for an error
+     *
+     * @param errorCode error code, cannot be SUCCESS
+     * @param extraInformation extra information
+     */
+    public ResolvedEntityResult(
+        @Nonnull BaseResult.ReturnStatus errorCode, @Nullable String extraInformation) {
+      super(errorCode, extraInformation);
+      this.entity = null;
+      this.entityGrantRecords = null;
+      this.grantRecordsVersion = 0;
+    }
+
+    /**
+     * Constructor with success
+     *
+     * @param entity the main entity
+     * @param grantRecordsVersion the version of the grant records
+     * @param entityGrantRecords the list of grant records
+     */
+    public ResolvedEntityResult(
+        @Nullable PolarisBaseEntity entity,
+        int grantRecordsVersion,
+        @Nullable List<PolarisGrantRecord> entityGrantRecords) {
+      super(BaseResult.ReturnStatus.SUCCESS);
+      this.entity = entity;
+      this.entityGrantRecords = entityGrantRecords;
+      this.grantRecordsVersion = grantRecordsVersion;
+    }
+
+    @JsonCreator
+    public ResolvedEntityResult(
+        @JsonProperty("returnStatus") @Nonnull BaseResult.ReturnStatus returnStatus,
+        @JsonProperty("extraInformation") String extraInformation,
+        @Nullable @JsonProperty("entity") PolarisBaseEntity entity,
+        @JsonProperty("grantRecordsVersion") int grantRecordsVersion,
+        @Nullable @JsonProperty("entityGrantRecords") List<PolarisGrantRecord> entityGrantRecords) {
+      super(returnStatus, extraInformation);
+      this.entity = entity;
+      this.entityGrantRecords = entityGrantRecords;
+      this.grantRecordsVersion = grantRecordsVersion;
+    }
+
+    public @Nullable PolarisBaseEntity getEntity() {
+      return entity;
+    }
+
+    public int getGrantRecordsVersion() {
+      return grantRecordsVersion;
+    }
+
+    public @Nullable List<PolarisGrantRecord> getEntityGrantRecords() {
+      return entityGrantRecords;
+    }
+  }
+
+  /**
+   * Load a resolved entity, i.e. an entity definition and associated grant records, from the
+   * backend store. The entity is identified by its id (entity catalog id and id).
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityCatalogId id of the catalog for that entity
+   * @param entityId id of the entity
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult loadResolvedEntityById(
+      @Nonnull PolarisCallContext callCtx, long entityCatalogId, long entityId);
+
+  /**
+   * Load a resolved entity, i.e. an entity definition and associated grant records, from the
+   * backend store. The entity is identified by its name. Will return NULL if the entity does not
+   * exist, i.e. has been purged or dropped.
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityCatalogId id of the catalog for that entity
+   * @param parentId the id of the parent of that entity
+   * @param entityType the type of this entity
+   * @param entityName the name of this entity
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult loadResolvedEntityByName(
+      @Nonnull PolarisCallContext callCtx,
+      long entityCatalogId,
+      long parentId,
+      @Nonnull PolarisEntityType entityType,
+      @Nonnull String entityName);
+
+  /**
+   * Refresh a resolved entity from the backend store. Will return NULL if the entity does not
+   * exist, i.e. has been purged or dropped. Else, will determine what has changed based on the
+   * version information sent by the caller and will return only what has changed.
+   *
+   * <p>For entities that can be grantees, the associated grant records will include both the grant
+   * records for this entity as a grantee and for this entity as a securable.
+   *
+   * @param callCtx call context
+   * @param entityType type of the entity whose entity and grants we are refreshing
+   * @param entityCatalogId id of the catalog for that entity
+   * @param entityId the id of the entity to load
+   * @return result with entity and grants. Status will be ENTITY_NOT_FOUND if the entity was not
+   *     found
+   */
+  @Nonnull
+  ResolvedEntityResult refreshResolvedEntity(
+      @Nonnull PolarisCallContext callCtx,
+      int entityVersion,
+      int entityGrantRecordsVersion,
+      @Nonnull PolarisEntityType entityType,
+      long entityCatalogId,
+      long entityId);
 }