diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/GenericTableEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/GenericTableEntity.java new file mode 100644 index 0000000000..5553b1a46c --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/GenericTableEntity.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.entity; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; + +/** + * A {@link PolarisEntity} implementation for generic tables. These tables are not Iceberg-like in + * that they may not have a schema or base location. Similarly to {@link IcebergTableLikeEntity} + * however, these tables have an identifier and a parent namespace. + */ +public class GenericTableEntity extends PolarisEntity { + + public static final String FORMAT_KEY = "format"; + + public GenericTableEntity(PolarisBaseEntity sourceEntity) { + super(sourceEntity); + } + + public static GenericTableEntity of(PolarisBaseEntity sourceEntity) { + if (sourceEntity != null) { + return new GenericTableEntity(sourceEntity); + } + return null; + } + + @JsonIgnore + public String getFormat() { + return getInternalPropertiesAsMap().get(GenericTableEntity.FORMAT_KEY); + } + + public static class Builder + extends PolarisEntity.BaseBuilder { + public Builder(TableIdentifier tableIdentifier, String format) { + super(); + setType(PolarisEntityType.GENERIC_TABLE); + setTableIdentifier(tableIdentifier); + setFormat(format); + } + + public GenericTableEntity.Builder setFormat(String format) { + // TODO in the future, we may validate the format and require certain properties + internalProperties.put(GenericTableEntity.FORMAT_KEY, format); + return this; + } + + public GenericTableEntity.Builder setTableIdentifier(TableIdentifier identifier) { + Namespace namespace = identifier.namespace(); + setParentNamespace(namespace); + setName(identifier.name()); + return this; + } + + public GenericTableEntity.Builder setParentNamespace(Namespace namespace) { + if (namespace != null && !namespace.isEmpty()) { + internalProperties.put( + NamespaceEntity.PARENT_NAMESPACE_KEY, RESTUtil.encodeNamespace(namespace)); + } + return this; + } + + @Override + public GenericTableEntity build() { + return new GenericTableEntity(buildBase()); + } + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/IcebergTableLikeEntity.java similarity index 88% rename from polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java rename to polaris-core/src/main/java/org/apache/polaris/core/entity/IcebergTableLikeEntity.java index 968598b93e..e7e1810754 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/TableLikeEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/IcebergTableLikeEntity.java @@ -24,7 +24,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTUtil; -public class TableLikeEntity extends PolarisEntity { +public class IcebergTableLikeEntity extends PolarisEntity { // For applicable types, this key on the "internalProperties" map will return the location // of the internalProperties JSON file. public static final String METADATA_LOCATION_KEY = "metadata-location"; @@ -35,13 +35,13 @@ public class TableLikeEntity extends PolarisEntity { public static final String LAST_ADMITTED_NOTIFICATION_TIMESTAMP_KEY = "last-notification-timestamp"; - public TableLikeEntity(PolarisBaseEntity sourceEntity) { + public IcebergTableLikeEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); } - public static TableLikeEntity of(PolarisBaseEntity sourceEntity) { + public static IcebergTableLikeEntity of(PolarisBaseEntity sourceEntity) { if (sourceEntity != null) { - return new TableLikeEntity(sourceEntity); + return new IcebergTableLikeEntity(sourceEntity); } return null; } @@ -79,21 +79,21 @@ public String getBaseLocation() { return getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION); } - public static class Builder extends PolarisEntity.BaseBuilder { + public static class Builder extends PolarisEntity.BaseBuilder { public Builder(TableIdentifier identifier, String metadataLocation) { super(); - setType(PolarisEntityType.TABLE_LIKE); + setType(PolarisEntityType.ICEBERG_TABLE_LIKE); setTableIdentifier(identifier); setMetadataLocation(metadataLocation); } - public Builder(TableLikeEntity original) { + public Builder(IcebergTableLikeEntity original) { super(original); } @Override - public TableLikeEntity build() { - return new TableLikeEntity(buildBase()); + public IcebergTableLikeEntity build() { + return new IcebergTableLikeEntity(buildBase()); } public Builder setTableIdentifier(TableIdentifier identifier) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisBaseEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisBaseEntity.java index 148733f52d..b9b4e1c006 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisBaseEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisBaseEntity.java @@ -28,7 +28,7 @@ /** * Base polaris entity representing all attributes of a Polaris Entity. This is used to exchange - * full entity information between the client and the GS backend + * full entity information between the client and the backend */ public class PolarisBaseEntity extends PolarisEntityCore { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java index c206d07894..e45f4d910f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntitySubType.java @@ -30,8 +30,8 @@ public enum PolarisEntitySubType { ANY_SUBTYPE(-1, null), // the NULL value is used when an entity has no subtype, i.e. NOT_APPLICABLE really NULL_SUBTYPE(0, null), - TABLE(2, PolarisEntityType.TABLE_LIKE), - VIEW(3, PolarisEntityType.TABLE_LIKE); + TABLE(2, PolarisEntityType.ICEBERG_TABLE_LIKE), + VIEW(3, PolarisEntityType.ICEBERG_TABLE_LIKE); // to efficiently map the code of a subtype to its corresponding subtype enum, use a reverse // array which is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java index 4a3eada34a..90d9aa9e5f 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisEntityType.java @@ -32,10 +32,11 @@ public enum PolarisEntityType { CATALOG_ROLE(5, CATALOG, true, false), NAMESPACE(6, CATALOG, false, true), // generic table is either a view or a real table - TABLE_LIKE(7, NAMESPACE, false, false), + ICEBERG_TABLE_LIKE(7, NAMESPACE, false, false), TASK(8, ROOT, false, false), - FILE(9, TABLE_LIKE, false, false), - POLICY(10, NAMESPACE, false, false); + FILE(9, ICEBERG_TABLE_LIKE, false, false), + POLICY(10, NAMESPACE, false, false), + GENERIC_TABLE(11, NAMESPACE, false, false); // to efficiently map a code to its corresponding entity type, use a reverse array which // is initialized below diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java index b674af2678..122d39a4ce 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java @@ -41,22 +41,22 @@ public enum PolarisPrivilege { TABLE_CREATE(6, PolarisEntityType.NAMESPACE), VIEW_CREATE(7, PolarisEntityType.NAMESPACE), NAMESPACE_DROP(8, PolarisEntityType.NAMESPACE), - TABLE_DROP(9, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_DROP(10, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_DROP(9, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_DROP(10, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), NAMESPACE_LIST(11, PolarisEntityType.NAMESPACE), TABLE_LIST(12, PolarisEntityType.NAMESPACE), VIEW_LIST(13, PolarisEntityType.NAMESPACE), NAMESPACE_READ_PROPERTIES(14, PolarisEntityType.NAMESPACE), - TABLE_READ_PROPERTIES(15, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_READ_PROPERTIES(16, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_READ_PROPERTIES(15, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_READ_PROPERTIES(16, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), NAMESPACE_WRITE_PROPERTIES(17, PolarisEntityType.NAMESPACE), - TABLE_WRITE_PROPERTIES(18, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_WRITE_PROPERTIES(19, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), - TABLE_READ_DATA(20, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - TABLE_WRITE_DATA(21, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), + TABLE_WRITE_PROPERTIES(18, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_WRITE_PROPERTIES(19, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_READ_DATA(20, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + TABLE_WRITE_DATA(21, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), NAMESPACE_FULL_METADATA(22, PolarisEntityType.NAMESPACE), - TABLE_FULL_METADATA(23, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_FULL_METADATA(24, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_FULL_METADATA(23, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_FULL_METADATA(24, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), CATALOG_CREATE(25, PolarisEntityType.ROOT), CATALOG_DROP(26, PolarisEntityType.CATALOG), CATALOG_LIST(27, PolarisEntityType.ROOT), @@ -70,12 +70,14 @@ public enum PolarisPrivilege { CATALOG_ROLE_LIST_GRANTS(35, PolarisEntityType.PRINCIPAL), CATALOG_LIST_GRANTS(36, PolarisEntityType.CATALOG), NAMESPACE_LIST_GRANTS(37, PolarisEntityType.NAMESPACE), - TABLE_LIST_GRANTS(38, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_LIST_GRANTS(39, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_LIST_GRANTS(38, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_LIST_GRANTS(39, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), CATALOG_MANAGE_GRANTS_ON_SECURABLE(40, PolarisEntityType.CATALOG), NAMESPACE_MANAGE_GRANTS_ON_SECURABLE(41, PolarisEntityType.NAMESPACE), - TABLE_MANAGE_GRANTS_ON_SECURABLE(42, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE), - VIEW_MANAGE_GRANTS_ON_SECURABLE(43, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW), + TABLE_MANAGE_GRANTS_ON_SECURABLE( + 42, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE), + VIEW_MANAGE_GRANTS_ON_SECURABLE( + 43, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW), PRINCIPAL_CREATE(44, PolarisEntityType.ROOT), PRINCIPAL_DROP(45, PolarisEntityType.PRINCIPAL), PRINCIPAL_LIST(46, PolarisEntityType.ROOT), diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java index d37e2ce2ff..fd000a167a 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifest.java @@ -180,8 +180,9 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key) { * "optional", or if it was resolved but the subType doesn't match the specified subType. */ @Override - public PolarisResolvedPathWrapper getResolvedPath(Object key, PolarisEntitySubType subType) { - return getResolvedPath(key, subType, false); + public PolarisResolvedPathWrapper getResolvedPath( + Object key, PolarisEntityType entityType, PolarisEntitySubType subType) { + return getResolvedPath(key, entityType, subType, false); } /** @@ -237,7 +238,7 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { */ @Override public PolarisResolvedPathWrapper getPassthroughResolvedPath( - Object key, PolarisEntitySubType subType) { + Object key, PolarisEntityType entityType, PolarisEntitySubType subType) { PolarisResolvedPathWrapper resolvedPath = getPassthroughResolvedPath(key); if (resolvedPath == null) { return null; @@ -374,7 +375,10 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key, boolean prependRoo * "optional", or if it was resolved but the subType doesn't match the specified subType. */ public PolarisResolvedPathWrapper getResolvedPath( - Object key, PolarisEntitySubType subType, boolean prependRootContainer) { + Object key, + PolarisEntityType entityType, + PolarisEntitySubType subType, + boolean prependRootContainer) { PolarisResolvedPathWrapper resolvedPath = getResolvedPath(key, prependRootContainer); if (resolvedPath == null) { return null; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifestCatalogView.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifestCatalogView.java index 21e16f5757..e3d91cad12 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifestCatalogView.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/PolarisResolutionManifestCatalogView.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.persistence.resolver; import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; /** @@ -30,9 +31,11 @@ public interface PolarisResolutionManifestCatalogView { PolarisResolvedPathWrapper getResolvedPath(Object key); - PolarisResolvedPathWrapper getResolvedPath(Object key, PolarisEntitySubType subType); + PolarisResolvedPathWrapper getResolvedPath( + Object key, PolarisEntityType entityType, PolarisEntitySubType subType); PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key); - PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key, PolarisEntitySubType subType); + PolarisResolvedPathWrapper getPassthroughResolvedPath( + Object key, PolarisEntityType entityType, PolarisEntitySubType subType); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java index b1c7fc1697..7e152c05f6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/Resolver.java @@ -373,7 +373,8 @@ public ResolverStatus resolveAll() { // validate input diagnostics.check( - entityType != PolarisEntityType.NAMESPACE && entityType != PolarisEntityType.TABLE_LIKE, + entityType != PolarisEntityType.NAMESPACE + && entityType != PolarisEntityType.ICEBERG_TABLE_LIKE, "cannot_be_path"); diagnostics.check( entityType.isTopLevel() || this.referenceCatalogName != null, "reference_catalog_expected"); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolverPath.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolverPath.java index ca349ccaa9..cf092d2c8b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolverPath.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/resolver/ResolverPath.java @@ -29,7 +29,7 @@ public class ResolverPath { // catalog private final List entityNames; - // all entities in a path are namespaces except the last one which can be a table_like entity + // all entities in a path are namespaces except the last one which can be a table_like entity // versus a namespace private final PolarisEntityType lastEntityType; @@ -39,9 +39,17 @@ public class ResolverPath { /** * Constructor for an optional path * - * @param entityNames set of entity names, all are namespaces except the last one which is either - * a namespace or a table_like entity - * @param lastEntityType type of the last entity, either namespace or table_like + * @param entityNames set of entity names, all are namespaces except the last one a namespa + */ + public ResolverPath(List entityNames) { + this(entityNames, null, false); + } + + /** + * Constructor for an optional path + * + * @param entityNames set of entity names, all are namespaces except the last one + * @param lastEntityType type of the last entity */ public ResolverPath(List entityNames, PolarisEntityType lastEntityType) { this(entityNames, lastEntityType, false); @@ -50,9 +58,8 @@ public ResolverPath(List entityNames, PolarisEntityType lastEntityType) /** * Constructor for an optional path * - * @param entityNames set of entity names, all are namespaces except the last one which is either - * a namespace or a table_like entity - * @param lastEntityType type of the last entity, either namespace or table_like + * @param entityNames set of entity names, all are namespaces except the last one + * @param lastEntityType type of the last entity * @param isOptional true if optional */ public ResolverPath( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java index 00f6ef6be6..161af2884c 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java @@ -41,9 +41,9 @@ import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; @@ -198,8 +198,8 @@ private static List userSpecifiedWriteLocations(Map prop .map( p -> Stream.of( - p.get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), - p.get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) + p.get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), + p.get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) .filter(Objects::nonNull) .collect(Collectors.toList())) .orElse(List.of()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java index 3fe5931950..20a371f4e0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/cache/StorageCredentialCache.java @@ -154,7 +154,7 @@ public Map getIfPresent(StorageCredentialCacheKey key) { private boolean isTypeSupported(PolarisEntityType type) { return type == PolarisEntityType.CATALOG || type == PolarisEntityType.NAMESPACE - || type == PolarisEntityType.TABLE_LIKE + || type == PolarisEntityType.ICEBERG_TABLE_LIKE || type == PolarisEntityType.TASK; } diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java index 0a3986357c..a086408da7 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/EntityCacheTest.java @@ -298,7 +298,7 @@ void testRefresh() { PolarisBaseEntity T6v1 = this.tm.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T6"); Assertions.assertThat(T6v1).isNotNull(); @@ -435,7 +435,8 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { Assertions.assertThat(lookup).isNotNull(); EntityCacheByNameKey T4_name = - new EntityCacheByNameKey(N1.getCatalogId(), N1.getId(), PolarisEntityType.TABLE_LIKE, "T4"); + new EntityCacheByNameKey( + N1.getCatalogId(), N1.getId(), PolarisEntityType.ICEBERG_TABLE_LIKE, "T4"); lookup = cache.getOrLoadEntityByName(callCtx, T4_name); Assertions.assertThat(lookup).isNotNull(); ResolvedPolarisEntity cacheEntry_T4 = lookup.getCacheEntry(); @@ -450,7 +451,7 @@ void testRenameAndCacheDestinationBeforeLoadingSource() { // load the renamed entity into cache EntityCacheByNameKey T4_renamed = new EntityCacheByNameKey( - N1.getCatalogId(), N1.getId(), PolarisEntityType.TABLE_LIKE, "T4_renamed"); + N1.getCatalogId(), N1.getId(), PolarisEntityType.ICEBERG_TABLE_LIKE, "T4_renamed"); lookup = cache.getOrLoadEntityByName(callCtx, T4_renamed); Assertions.assertThat(lookup).isNotNull(); ResolvedPolarisEntity cacheEntry_T4_renamed = lookup.getCacheEntry(); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java index dd21a16f3b..2b6e8423c8 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/ResolverTest.java @@ -216,12 +216,12 @@ void testResolvePath(boolean useCache) { // N1/N2/T1 which exists ResolverPath N1_N2_T1 = - new ResolverPath(List.of("N1", "N2", "T1"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N1", "N2", "T1"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver(this.cache, "test", N1_N2_T1, null, null); // N1/N2/T1 which exists ResolverPath N1_N2_V1 = - new ResolverPath(List.of("N1", "N2", "V1"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N1", "N2", "V1"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver(this.cache, "test", N1_N2_V1, null, null); // N5/N6 which exists @@ -230,7 +230,7 @@ void testResolvePath(boolean useCache) { // N5/N6/T5 which exists ResolverPath N5_N6_T5 = - new ResolverPath(List.of("N5", "N6", "T5"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N5", "N6", "T5"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver(this.cache, "test", N5_N6_T5, null, null); // N7/N8 which exists @@ -248,7 +248,7 @@ void testResolvePath(boolean useCache) { // Error scenarios: N5/N6/T8 which does not exists ResolverPath N5_N6_T8 = - new ResolverPath(List.of("N5", "N6", "T8"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N5", "N6", "T8"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver( this.cache, "test", @@ -258,7 +258,7 @@ void testResolvePath(boolean useCache) { // Error scenarios: N8/N6/T8 which does not exists ResolverPath N8_N6_T8 = - new ResolverPath(List.of("N8", "N6", "T8"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N8", "N6", "T8"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver( this.cache, "test", @@ -277,7 +277,8 @@ void testResolvePath(boolean useCache) { ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED); // except if the optional flag is specified - N5_N6_T8 = new ResolverPath(List.of("N5", "N6", "T8"), PolarisEntityType.TABLE_LIKE, true); + N5_N6_T8 = + new ResolverPath(List.of("N5", "N6", "T8"), PolarisEntityType.ICEBERG_TABLE_LIKE, true); Resolver resolver = this.resolveDriver(this.cache, "test", null, List.of(N1, N5_N6_T8, N5_N6_T5, N1_N2), null); // get all the resolved paths @@ -365,7 +366,7 @@ void testPathConsistency(boolean useCache) { ResolverPath N1_N2_PATH = new ResolverPath(List.of("N1", "N2"), PolarisEntityType.NAMESPACE); this.resolveDriver(this.cache, "test", N1_N2_PATH, null, null); ResolverPath N1_N2_T1_PATH = - new ResolverPath(List.of("N1", "N2", "T1"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N1", "N2", "T1"), PolarisEntityType.ICEBERG_TABLE_LIKE); Resolver resolver = this.resolveDriver(this.cache, "test", N1_N2_T1_PATH, null, null); // get the catalog @@ -399,7 +400,7 @@ void testPathConsistency(boolean useCache) { // but we should be able to resolve it under N1/N3 ResolverPath N1_N3_T1_PATH = - new ResolverPath(List.of("N1", "N3", "T1"), PolarisEntityType.TABLE_LIKE); + new ResolverPath(List.of("N1", "N3", "T1"), PolarisEntityType.ICEBERG_TABLE_LIKE); this.resolveDriver(this.cache, "test", N1_N3_T1_PATH, null, null); } diff --git a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java index 4d1927a444..b8470dee94 100644 --- a/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java +++ b/polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java @@ -742,7 +742,7 @@ void dropEntity(List catalogPath, PolarisBaseEntity entityToD .listEntities( this.polarisCallContext, path, - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE) .getEntities(); Assertions.assertThat(children).isNotNull(); @@ -1003,27 +1003,36 @@ PolarisBaseEntity createTestCatalog(String catalogName) { this.createEntity(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N2"); this.createEntity( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T1"); this.createEntity( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T2"); this.createEntity( - List.of(catalog, N1, N1_N2), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW, "V1"); + List.of(catalog, N1, N1_N2), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.VIEW, + "V1"); PolarisBaseEntity N1_N3 = this.createEntity(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N3"); this.createEntity( List.of(catalog, N1, N1_N3), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T3"); this.createEntity( - List.of(catalog, N1, N1_N3), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW, "V2"); + List.of(catalog, N1, N1_N3), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.VIEW, + "V2"); this.createEntity( - List.of(catalog, N1), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE, "T4"); + List.of(catalog, N1), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.TABLE, + "T4"); this.createEntity(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N4"); PolarisBaseEntity N5 = this.createEntity(List.of(catalog), PolarisEntityType.NAMESPACE, "N5"); PolarisBaseEntity N5_N6 = @@ -1031,12 +1040,12 @@ PolarisBaseEntity createTestCatalog(String catalogName) { PolarisBaseEntity N5_N6_T5 = this.createEntity( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T5"); this.createEntity( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T6"); @@ -1655,40 +1664,46 @@ void testCreateTestCatalog() { this.ensureExistsByName(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N2"); this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T1"); this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T2"); this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "T2"); this.ensureExistsByName( - List.of(catalog, N1, N1_N2), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.VIEW, "V1"); + List.of(catalog, N1, N1_N2), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.VIEW, + "V1"); this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "V1"); PolarisBaseEntity N1_N3 = this.ensureExistsByName(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N3"); this.ensureExistsByName( List.of(catalog, N1, N1_N3), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T3"); this.ensureExistsByName( List.of(catalog, N1, N1_N3), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "V2"); this.ensureExistsByName( - List.of(catalog, N1), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE, "T4"); + List.of(catalog, N1), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.TABLE, + "T4"); this.ensureExistsByName(List.of(catalog, N1), PolarisEntityType.NAMESPACE, "N4"); PolarisBaseEntity N5 = this.ensureExistsByName(List.of(catalog), PolarisEntityType.NAMESPACE, "N5"); @@ -1700,18 +1715,18 @@ void testCreateTestCatalog() { "N6"); this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T5"); PolarisBaseEntity N5_N6_T5 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "T5"); this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T6"); PolarisBaseEntity N7 = @@ -1806,7 +1821,7 @@ void testBrowse() { ImmutablePair.of("N4", PolarisEntitySubType.NULL_SUBTYPE))); this.validateListReturn( List.of(catalog, N1), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, List.of(ImmutablePair.of("T4", PolarisEntitySubType.TABLE))); PolarisBaseEntity N5 = @@ -1822,7 +1837,7 @@ void testBrowse() { // table or view object this.validateListReturn( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, List.of( ImmutablePair.of("T1", PolarisEntitySubType.TABLE), @@ -1831,7 +1846,7 @@ void testBrowse() { // table object only this.validateListReturn( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, List.of( ImmutablePair.of("T1", PolarisEntitySubType.TABLE), @@ -1839,7 +1854,7 @@ void testBrowse() { // view object only this.validateListReturn( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW, List.of(ImmutablePair.of("V1", PolarisEntitySubType.VIEW))); // list all principals @@ -1888,7 +1903,7 @@ void testUpdateEntities() { PolarisBaseEntity T6v1 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T6"); Assertions.assertThat(T6v1).isNotNull(); @@ -1925,7 +1940,7 @@ void testUpdateEntities() { PolarisBaseEntity T5v1 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T5"); T5v1.setId(100000L); @@ -1966,7 +1981,7 @@ void testDropEntities() { PolarisBaseEntity T6 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T6"); Assertions.assertThat(T6).isNotNull(); @@ -1986,21 +2001,21 @@ void testDropEntities() { PolarisBaseEntity T1 = this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T1"); this.dropEntity(List.of(catalog, N1, N1_N2), T1); PolarisBaseEntity T2 = this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T2"); this.dropEntity(List.of(catalog, N1, N1_N2), T2); PolarisBaseEntity V1 = this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW, "V1"); this.dropEntity(List.of(catalog, N1, N1_N2), V1); @@ -2011,14 +2026,14 @@ void testDropEntities() { PolarisBaseEntity T3 = this.ensureExistsByName( List.of(catalog, N1, N1_N3), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T3"); this.dropEntity(List.of(catalog, N1, N1_N3), T3); PolarisBaseEntity V2 = this.ensureExistsByName( List.of(catalog, N1, N1_N3), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW, "V2"); this.dropEntity(List.of(catalog, N1, N1_N3), V2); @@ -2026,14 +2041,17 @@ void testDropEntities() { PolarisBaseEntity T4 = this.ensureExistsByName( - List.of(catalog, N1), PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.TABLE, "T4"); + List.of(catalog, N1), + PolarisEntityType.ICEBERG_TABLE_LIKE, + PolarisEntitySubType.TABLE, + "T4"); this.dropEntity(List.of(catalog, N1), T4); this.dropEntity(List.of(catalog), N1); PolarisBaseEntity T5 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE, "T5"); this.dropEntity(List.of(catalog, N5, N5_N6), T5); @@ -2153,7 +2171,7 @@ public void testPrivileges() { PolarisBaseEntity N5_N6_T5 = this.ensureExistsByName( List.of(catalog, N5, N5_N6), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "T5"); @@ -2323,7 +2341,7 @@ public void testRename() { PolarisBaseEntity N1_N2_T1 = this.ensureExistsByName( List.of(catalog, N1, N1_N2), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE, "T1"); // view with the same name exists, should fail @@ -2418,11 +2436,15 @@ public void testEntityCache() { // now validate that load something which does not exist, will also work this.loadCacheEntryByName( - N1.getCatalogId(), N1.getId(), PolarisEntityType.TABLE_LIKE, "do_not_exists", false); + N1.getCatalogId(), + N1.getId(), + PolarisEntityType.ICEBERG_TABLE_LIKE, + "do_not_exists", + false); this.loadCacheEntryById(N1.getCatalogId() + 1000, N1.getId(), N1.getType(), false); // refresh a purged entity this.refreshCacheEntry( - 1, 1, PolarisEntityType.TABLE_LIKE, N1.getCatalogId() + 1000, N1.getId(), false); + 1, 1, PolarisEntityType.ICEBERG_TABLE_LIKE, N1.getCatalogId() + 1000, N1.getId(), false); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index ebf113ab70..6332edf703 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -75,8 +75,8 @@ import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisAdminService; -import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; import org.apache.polaris.service.config.RealmEntityManagerFactory; @@ -174,7 +174,7 @@ public Map getConfigOverrides() { @Inject protected Clock clock; @Inject protected FileIOFactory fileIOFactory; - protected BasePolarisCatalog baseCatalog; + protected IcebergCatalog baseCatalog; protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; protected PolarisMetaStoreManager metaStoreManager; @@ -430,7 +430,7 @@ private void initBaseCatalog() { new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, CATALOG_NAME); this.baseCatalog = - new BasePolarisCatalog( + new IcebergCatalog( entityManager, metaStoreManager, callContext, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogNoEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java similarity index 90% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogNoEntityCacheTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java index 5389dcd826..7e92b97b5a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogNoEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/CatalogNoEntityCacheTest.java @@ -25,8 +25,8 @@ import org.apache.polaris.core.persistence.cache.EntityCache; @QuarkusTest -@TestProfile(BasePolarisCatalogTest.Profile.class) -public class PolarisCatalogNoEntityCacheTest extends BasePolarisCatalogTest { +@TestProfile(IcebergCatalogTest.Profile.class) +public class CatalogNoEntityCacheTest extends IcebergCatalogTest { @Nullable @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java new file mode 100644 index 0000000000..57d49226c4 --- /dev/null +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.catalog; + +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.when; + +import io.quarkus.test.junit.QuarkusMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.SecurityContext; +import java.io.IOException; +import java.lang.reflect.Method; +import java.time.Clock; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.polaris.core.PolarisCallContext; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.auth.AuthenticatedPolarisPrincipal; +import org.apache.polaris.core.auth.PolarisAuthorizerImpl; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.config.PolarisConfigurationStore; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.GenericTableEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.PrincipalEntity; +import org.apache.polaris.core.persistence.BasePersistence; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisEntityManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.bootstrap.RootCredentialsSet; +import org.apache.polaris.core.persistence.cache.EntityCache; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; +import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; +import org.apache.polaris.service.catalog.generic.GenericTableCatalog; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; +import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; +import org.apache.polaris.service.task.TaskExecutor; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.mockito.Mockito; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; + +@QuarkusTest +public class GenericTableCatalogTest { + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of(); + } + } + + protected static final Namespace NS = Namespace.of("newdb"); + protected static final TableIdentifier TABLE = TableIdentifier.of(NS, "table"); + public static final String CATALOG_NAME = "polaris-catalog"; + public static final String TEST_ACCESS_KEY = "test_access_key"; + public static final String SECRET_ACCESS_KEY = "secret_access_key"; + public static final String SESSION_TOKEN = "session_token"; + + @Inject MetaStoreManagerFactory managerFactory; + @Inject PolarisConfigurationStore configurationStore; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject PolarisDiagnostics diagServices; + + private GenericTableCatalog genericTableCatalog; + private IcebergCatalog icebergCatalog; + private CallContext callContext; + private AwsStorageConfigInfo storageConfigModel; + private String realmName; + private PolarisMetaStoreManager metaStoreManager; + private PolarisCallContext polarisContext; + private PolarisAdminService adminService; + private PolarisEntityManager entityManager; + private FileIOFactory fileIOFactory; + private AuthenticatedPolarisPrincipal authenticatedRoot; + private PolarisEntity catalogEntity; + private SecurityContext securityContext; + + @BeforeAll + public static void setUpMocks() { + PolarisStorageIntegrationProviderImpl mock = + Mockito.mock(PolarisStorageIntegrationProviderImpl.class); + QuarkusMock.installMockForType(mock, PolarisStorageIntegrationProviderImpl.class); + } + + @BeforeEach + @SuppressWarnings("unchecked") + public void before(TestInfo testInfo) { + realmName = + "realm_%s_%s" + .formatted( + testInfo.getTestMethod().map(Method::getName).orElse("test"), System.nanoTime()); + RealmContext realmContext = () -> realmName; + metaStoreManager = managerFactory.getOrCreateMetaStoreManager(realmContext); + polarisContext = + new PolarisCallContext( + managerFactory.getOrCreateSessionSupplier(realmContext).get(), + diagServices, + configurationStore, + Clock.systemDefaultZone()); + entityManager = + new PolarisEntityManager( + metaStoreManager, new StorageCredentialCache(), new EntityCache(metaStoreManager)); + + callContext = CallContext.of(realmContext, polarisContext); + + PrincipalEntity rootEntity = + new PrincipalEntity( + PolarisEntity.of( + metaStoreManager + .readEntityByName( + polarisContext, + null, + PolarisEntityType.PRINCIPAL, + PolarisEntitySubType.NULL_SUBTYPE, + "root") + .getEntity())); + + authenticatedRoot = new AuthenticatedPolarisPrincipal(rootEntity, Set.of()); + + securityContext = Mockito.mock(SecurityContext.class); + when(securityContext.getUserPrincipal()).thenReturn(authenticatedRoot); + when(securityContext.isUserInRole(isA(String.class))).thenReturn(true); + adminService = + new PolarisAdminService( + callContext, + entityManager, + metaStoreManager, + securityContext, + new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); + + String storageLocation = "s3://my-bucket/path/to/data"; + storageConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::012345678901:role/jdoe") + .setExternalId("externalId") + .setUserArn("aws::a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) + .build(); + catalogEntity = + adminService.createCatalog( + new CatalogEntity.Builder() + .setName(CATALOG_NAME) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + FeatureConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build()); + + PolarisPassthroughResolutionView passthroughView = + new PolarisPassthroughResolutionView( + callContext, entityManager, securityContext, CATALOG_NAME); + TaskExecutor taskExecutor = Mockito.mock(); + + StsClient stsClient = Mockito.mock(StsClient.class); + when(stsClient.assumeRole(isA(AssumeRoleRequest.class))) + .thenReturn( + AssumeRoleResponse.builder() + .credentials( + Credentials.builder() + .accessKeyId(TEST_ACCESS_KEY) + .secretAccessKey(SECRET_ACCESS_KEY) + .sessionToken(SESSION_TOKEN) + .build()) + .build()); + PolarisStorageIntegration storageIntegration = + new AwsCredentialsStorageIntegration(stsClient); + when(storageIntegrationProvider.getStorageIntegrationForConfig( + isA(AwsStorageConfigurationInfo.class))) + .thenReturn((PolarisStorageIntegration) storageIntegration); + + this.genericTableCatalog = + new GenericTableCatalog( + entityManager, + metaStoreManager, + callContext, + passthroughView, + securityContext, + taskExecutor, + fileIOFactory); + this.icebergCatalog = + new IcebergCatalog( + entityManager, + metaStoreManager, + callContext, + passthroughView, + securityContext, + taskExecutor, + fileIOFactory); + } + + @AfterEach + public void after() throws IOException { + metaStoreManager.purge(polarisContext); + } + + private MetaStoreManagerFactory createMockMetaStoreManagerFactory() { + return new MetaStoreManagerFactory() { + @Override + public PolarisMetaStoreManager getOrCreateMetaStoreManager(RealmContext realmContext) { + return metaStoreManager; + } + + @Override + public Supplier getOrCreateSessionSupplier(RealmContext realmContext) { + return () -> polarisContext.getMetaStore(); + } + + @Override + public StorageCredentialCache getOrCreateStorageCredentialCache(RealmContext realmContext) { + return new StorageCredentialCache(); + } + + @Override + public EntityCache getOrCreateEntityCache(RealmContext realmContext) { + return new EntityCache(metaStoreManager); + } + + @Override + public Map bootstrapRealms( + Iterable realms, RootCredentialsSet rootCredentialsSet) { + throw new NotImplementedException("Bootstrapping realms is not supported"); + } + + @Override + public Map purgeRealms(Iterable realms) { + throw new NotImplementedException("Purging realms is not supported"); + } + }; + } + + @Test + public void testCreateGenericTableDoesNotThrow() { + Namespace namespace = Namespace.of("ns"); + icebergCatalog.createNamespace(namespace); + Assertions.assertThatCode( + () -> + genericTableCatalog.createGenericTable( + TableIdentifier.of("ns", "t1"), "test-format", Map.of())) + .doesNotThrowAnyException(); + } + + @Test + public void testGenericTableRoundTrip() { + Namespace namespace = Namespace.of("ns"); + icebergCatalog.createNamespace(namespace); + + String tableName = "t1"; + Map properties = Map.of("a", "b", "c", "d"); + String format = "round-trip-format"; + + genericTableCatalog.createGenericTable(TableIdentifier.of("ns", tableName), format, properties); + + GenericTableEntity resultEntity = + genericTableCatalog.loadGenericTable(TableIdentifier.of("ns", tableName)); + + Assertions.assertThat(resultEntity.getFormat()).isEqualTo(format); + Assertions.assertThat(resultEntity.getPropertiesAsMap()).isEqualTo(properties); + Assertions.assertThat(resultEntity.getName()).isEqualTo(tableName); + } +} diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java similarity index 99% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java index a17091cba2..527f23d0be 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java @@ -64,7 +64,7 @@ import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.service.catalog.PolarisCatalogHandlerWrapper; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalogHandlerWrapper; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.context.CallContextCatalogFactory; @@ -78,8 +78,8 @@ import org.mockito.Mockito; @QuarkusTest -@TestProfile(PolarisCatalogHandlerWrapperAuthzTest.Profile.class) -public class PolarisCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { +@TestProfile(IcebergCatalogHandlerWrapperAuthzTest.Profile.class) +public class IcebergCatalogHandlerWrapperAuthzTest extends PolarisAuthzTestBase { public static class Profile extends PolarisAuthzTestBase.Profile { @@ -93,19 +93,19 @@ public Map getConfigOverrides() { } } - private PolarisCatalogHandlerWrapper newWrapper() { + private IcebergCatalogHandlerWrapper newWrapper() { return newWrapper(Set.of()); } - private PolarisCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { + private IcebergCatalogHandlerWrapper newWrapper(Set activatedPrincipalRoles) { return newWrapper(activatedPrincipalRoles, CATALOG_NAME, callContextCatalogFactory); } - private PolarisCatalogHandlerWrapper newWrapper( + private IcebergCatalogHandlerWrapper newWrapper( Set activatedPrincipalRoles, String catalogName, CallContextCatalogFactory factory) { final AuthenticatedPolarisPrincipal authenticatedPrincipal = new AuthenticatedPolarisPrincipal(principalEntity, activatedPrincipalRoles); - return new PolarisCatalogHandlerWrapper( + return new IcebergCatalogHandlerWrapper( callContext, entityManager, metaStoreManager, @@ -242,8 +242,8 @@ public void testInsufficientPermissionsPriorToSecretRotation() { new AuthenticatedPolarisPrincipal( PrincipalEntity.of(newPrincipal.getPrincipal()), Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)); - PolarisCatalogHandlerWrapper wrapper = - new PolarisCatalogHandlerWrapper( + IcebergCatalogHandlerWrapper wrapper = + new IcebergCatalogHandlerWrapper( callContext, entityManager, metaStoreManager, @@ -274,8 +274,8 @@ public void testInsufficientPermissionsPriorToSecretRotation() { final AuthenticatedPolarisPrincipal authenticatedPrincipal1 = new AuthenticatedPolarisPrincipal( PrincipalEntity.of(refreshPrincipal), Set.of(PRINCIPAL_ROLE1, PRINCIPAL_ROLE2)); - PolarisCatalogHandlerWrapper refreshedWrapper = - new PolarisCatalogHandlerWrapper( + IcebergCatalogHandlerWrapper refreshedWrapper = + new IcebergCatalogHandlerWrapper( callContext, entityManager, metaStoreManager, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java similarity index 97% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 29401e2669..3dafaf109a 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.inject.Inject; @@ -106,8 +107,8 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; -import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.catalog.io.ExceptionMappingFileIO; import org.apache.polaris.service.catalog.io.FileIOFactory; @@ -140,7 +141,8 @@ import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; import software.amazon.awssdk.services.sts.model.Credentials; -public abstract class BasePolarisCatalogTest extends CatalogTests { +@TestProfile(IcebergCatalogTest.Profile.class) +public abstract class IcebergCatalogTest extends CatalogTests { public static class Profile implements QuarkusTestProfile { @@ -172,7 +174,7 @@ public Map getConfigOverrides() { @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; - private BasePolarisCatalog catalog; + private IcebergCatalog catalog; private CallContext callContext; private AwsStorageConfigInfo storageConfigModel; private StsClient stsClient; @@ -291,7 +293,7 @@ public void before(TestInfo testInfo) { .thenReturn((PolarisStorageIntegration) storageIntegration); this.catalog = - new BasePolarisCatalog( + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -312,7 +314,7 @@ public void after() throws IOException { } @Override - protected BasePolarisCatalog catalog() { + protected IcebergCatalog catalog() { return catalog; } @@ -377,7 +379,7 @@ public void testRenameTableMissingDestinationNamespace() { requiresNamespaceCreate(), "Only applicable if namespaces must be created before adding children"); - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); catalog.createNamespace(NS); Assertions.assertThat(catalog.tableExists(TABLE)) @@ -421,7 +423,7 @@ public void testCreateNestedNamespaceUnderMissingParent() { Assumptions.assumeTrue( supportsNestedNamespaces(), "Only applicable if nested namespaces are supoprted"); - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace child1 = Namespace.of("parent", "child1"); @@ -442,7 +444,7 @@ public void testValidateNotificationWhenTableAndNamespacesDontExist() { final String tableLocation = "s3://externally-owned-bucket/validate_table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -505,7 +507,7 @@ public void testValidateNotificationInDisallowedLocation() { // filename. final String tableLocation = "s3://forbidden-table-location/table/"; final String tableMetadataLocation = tableLocation + "metadata/"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -549,8 +551,8 @@ public void testValidateNotificationFailToCreateFileIO() { new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()), managerFactory, configurationStore)); - BasePolarisCatalog catalog = - new BasePolarisCatalog( + IcebergCatalog catalog = + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -595,7 +597,7 @@ public void testUpdateNotificationWhenTableAndNamespacesDontExist() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -639,7 +641,7 @@ public void testUpdateNotificationCreateTableInDisallowedLocation() { // The location of the metadata JSON file specified in the create will be forbidden. final String tableLocation = "s3://forbidden-table-location/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -690,7 +692,7 @@ public void testCreateNotificationCreateTableInExternalLocation() { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .build()); - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); TableMetadata tableMetadata = TableMetadata.buildFromEmpty() .assignUUID() @@ -747,7 +749,7 @@ public void testCreateNotificationCreateTableOutsideOfMetadataLocation() { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .build()); - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); TableMetadata tableMetadata = TableMetadata.buildFromEmpty() .assignUUID() @@ -801,7 +803,7 @@ public void testUpdateNotificationCreateTableInExternalLocation() { .addProperty( FeatureConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") .build()); - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); InMemoryFileIO fileIO = getInMemoryIo(catalog); fileIO.addFile( @@ -873,8 +875,8 @@ public void testUpdateNotificationCreateTableWithLocalFilePrefix() { new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, catalogWithoutStorage); TaskExecutor taskExecutor = Mockito.mock(); - BasePolarisCatalog catalog = - new BasePolarisCatalog( + IcebergCatalog catalog = + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -937,8 +939,8 @@ public void testUpdateNotificationCreateTableWithHttpPrefix() { new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, catalogName); TaskExecutor taskExecutor = Mockito.mock(); - BasePolarisCatalog catalog = - new BasePolarisCatalog( + IcebergCatalog catalog = + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -1015,7 +1017,7 @@ public void testUpdateNotificationWhenNamespacesExist() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1061,7 +1063,7 @@ public void testUpdateNotificationWhenTableExists() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1114,7 +1116,7 @@ public void testUpdateNotificationWhenTableExistsInDisallowedLocation() { // The location of the metadata JSON file specified in the update will be forbidden. final String tableLocation = "s3://forbidden-table-location/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1160,7 +1162,7 @@ public void testUpdateNotificationRejectOutOfOrderTimestamp() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -1225,7 +1227,7 @@ public void testUpdateNotificationWhenTableExistsFileSpecifiesDisallowedLocation final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1274,7 +1276,7 @@ public void testDropNotificationWhenTableAndNamespacesDontExist() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); TableIdentifier table = TableIdentifier.of(namespace, "table"); @@ -1309,7 +1311,7 @@ public void testDropNotificationWhenNamespacesExist() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1355,7 +1357,7 @@ public void testDropNotificationWhenTableExists() { final String tableLocation = "s3://externally-owned-bucket/table/"; final String tableMetadataLocation = tableLocation + "metadata/v1.metadata.json"; - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Namespace namespace = Namespace.of("parent", "child1"); @@ -1480,8 +1482,8 @@ public void testDropTableWithPurgeDisabled() { PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, noPurgeCatalogName); - BasePolarisCatalog noPurgeCatalog = - new BasePolarisCatalog( + IcebergCatalog noPurgeCatalog = + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -1538,7 +1540,7 @@ private void createNonExistingNamespaces(Namespace namespace) { @ParameterizedTest @MethodSource public void testRetriableException(Exception exception, boolean shouldRetry) { - Assertions.assertThat(BasePolarisCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(exception)) + Assertions.assertThat(IcebergCatalog.SHOULD_RETRY_REFRESH_PREDICATE.test(exception)) .isEqualTo(shouldRetry); } @@ -1588,8 +1590,8 @@ public void testFileIOWrapper() { new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()), managerFactory, configurationStore); - BasePolarisCatalog catalog = - new BasePolarisCatalog( + IcebergCatalog catalog = + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -1661,7 +1663,7 @@ public FileIO loadFileIO( @Test public void testRegisterTableWithSlashlessMetadataLocation() { - BasePolarisCatalog catalog = catalog(); + IcebergCatalog catalog = catalog(); Assertions.assertThatThrownBy( () -> catalog.registerTable(TABLE, "metadata_location_without_slashes")) .isInstanceOf(IllegalArgumentException.class) @@ -1685,8 +1687,8 @@ public void testConcurrencyConflictCreateTableUpdatedDuringFinalTransaction() { PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, CATALOG_NAME); - final BasePolarisCatalog catalog = - new BasePolarisCatalog( + final IcebergCatalog catalog = + new IcebergCatalog( entityManager, spyMetaStore, callContext, @@ -1733,8 +1735,8 @@ public void testConcurrencyConflictUpdateTableDuringFinalTransaction() { PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( callContext, entityManager, securityContext, CATALOG_NAME); - final BasePolarisCatalog catalog = - new BasePolarisCatalog( + final IcebergCatalog catalog = + new IcebergCatalog( entityManager, spyMetaStore, callContext, @@ -1766,7 +1768,7 @@ public void testConcurrencyConflictUpdateTableDuringFinalTransaction() { .hasMessageContaining("conflict_table"); } - private static InMemoryFileIO getInMemoryIo(BasePolarisCatalog catalog) { + private static InMemoryFileIO getInMemoryIo(IcebergCatalog catalog) { return (InMemoryFileIO) ((ExceptionMappingFileIO) catalog.getIo()).getInnerIo(); } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java similarity index 96% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 471afa2d9f..bea9aa3100 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -59,8 +59,8 @@ import org.apache.polaris.core.persistence.cache.EntityCache; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; -import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; @@ -73,8 +73,8 @@ import org.mockito.Mockito; @QuarkusTest -@TestProfile(BasePolarisCatalogViewTest.Profile.class) -public class BasePolarisCatalogViewTest extends ViewCatalogTests { +@TestProfile(IcebergCatalogViewTest.Profile.class) +public class IcebergCatalogViewTest extends ViewCatalogTests { public static class Profile implements QuarkusTestProfile { @@ -100,7 +100,7 @@ public Map getConfigOverrides() { @Inject PolarisConfigurationStore configurationStore; @Inject PolarisDiagnostics diagServices; - private BasePolarisCatalog catalog; + private IcebergCatalog catalog; private String realmName; private PolarisMetaStoreManager metaStoreManager; @@ -188,7 +188,7 @@ public void before(TestInfo testInfo) { new DefaultFileIOFactory( new RealmEntityManagerFactory(managerFactory), managerFactory, configurationStore); this.catalog = - new BasePolarisCatalog( + new IcebergCatalog( entityManager, metaStoreManager, callContext, @@ -209,7 +209,7 @@ public void after() throws IOException { } @Override - protected BasePolarisCatalog catalog() { + protected IcebergCatalog catalog() { return catalog; } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java index f408267d07..64c96bb5be 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogWithEntityCacheTest.java @@ -25,8 +25,8 @@ import org.apache.polaris.core.persistence.cache.EntityCache; @QuarkusTest -@TestProfile(BasePolarisCatalogTest.Profile.class) -public class PolarisCatalogWithEntityCacheTest extends BasePolarisCatalogTest { +@TestProfile(IcebergCatalogTest.Profile.class) +public class PolarisCatalogWithEntityCacheTest extends IcebergCatalogTest { @Nullable @Override diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java index f43603f42e..d723f6caf0 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TableCleanupTaskHandlerTest.java @@ -43,10 +43,10 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.BasePersistence; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; @@ -127,7 +127,7 @@ public void testTableCleanup() throws IOException { .setName("cleanup_" + tableIdentifier) .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) + new IcebergTableLikeEntity.Builder(tableIdentifier, metadataFile) .setName("table1") .setCatalogId(1) .setCreateTimestamp(100) @@ -196,8 +196,8 @@ public void close() { String metadataFile = "v1-49494949.metadata.json"; TaskTestUtils.writeTableMetadata(fileIO, metadataFile, snapshot); - TableLikeEntity tableLikeEntity = - new TableLikeEntity.Builder(tableIdentifier, metadataFile) + IcebergTableLikeEntity icebergTableLikeEntity = + new IcebergTableLikeEntity.Builder(tableIdentifier, metadataFile) .setName("table1") .setCatalogId(1) .setCreateTimestamp(100) @@ -206,7 +206,7 @@ public void close() { new TaskEntity.Builder() .setName("cleanup_" + tableIdentifier) .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) - .withData(tableLikeEntity) + .withData(icebergTableLikeEntity) .build(); addTaskLocation(task); Assertions.assertThatPredicate(handler::canHandleTask).accepts(task); @@ -263,7 +263,7 @@ public void close() { .setName("cleanup_" + tableIdentifier) .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) + new IcebergTableLikeEntity.Builder(tableIdentifier, metadataFile) .setName("table1") .setCatalogId(1) .setCreateTimestamp(100) @@ -388,7 +388,7 @@ public void testTableCleanupMultipleSnapshots() throws IOException { .setName("cleanup_" + tableIdentifier) .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) .withData( - new TableLikeEntity.Builder(tableIdentifier, metadataFile) + new IcebergTableLikeEntity.Builder(tableIdentifier, metadataFile) .setName("table1") .setCatalogId(1) .setCreateTimestamp(100) @@ -546,7 +546,7 @@ public void testTableCleanupMultipleMetadata() throws IOException { .setName("cleanup_" + tableIdentifier) .withTaskType(AsyncTaskType.ENTITY_CLEANUP_SCHEDULER) .withData( - new TableLikeEntity.Builder(tableIdentifier, secondMetadataFile) + new IcebergTableLikeEntity.Builder(tableIdentifier, secondMetadataFile) .setName("table1") .setCatalogId(1) .setCreateTimestamp(100) diff --git a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java index 76f5997bdc..bedb10165c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java +++ b/service/common/src/main/java/org/apache/polaris/service/admin/PolarisAdminService.java @@ -69,6 +69,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.CatalogRoleEntity; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; @@ -79,7 +80,6 @@ import org.apache.polaris.core.entity.PolarisPrivilege; import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.PrincipalRoleEntity; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; @@ -461,7 +461,8 @@ private void authorizeGrantOnTableLikeOperationOrThrow( entityManager.prepareResolutionManifest(callContext, securityContext, catalogName); resolutionManifest.addPath( new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), PolarisEntityType.TABLE_LIKE), + PolarisCatalogHelpers.tableIdentifierToList(identifier), + PolarisEntityType.ICEBERG_TABLE_LIKE), identifier); resolutionManifest.addPath( new ResolverPath(List.of(catalogRoleName), PolarisEntityType.CATALOG_ROLE), @@ -471,7 +472,8 @@ private void authorizeGrantOnTableLikeOperationOrThrow( if (status.getStatus() == ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED) { throw new NotFoundException("Catalog not found: %s", catalogName); } else if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED) { - if (status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.TABLE_LIKE) { + if (status.getFailedToResolvePath().getLastEntityType() + == PolarisEntityType.ICEBERG_TABLE_LIKE) { if (subType == PolarisEntitySubType.TABLE) { throw new NoSuchTableException("Table does not exist: %s", identifier); } else { @@ -483,7 +485,8 @@ private void authorizeGrantOnTableLikeOperationOrThrow( } PolarisResolvedPathWrapper tableLikeWrapper = - resolutionManifest.getResolvedPath(identifier, subType, true); + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType, true); PolarisResolvedPathWrapper catalogRoleWrapper = resolutionManifest.getResolvedPath(catalogRoleName, true); @@ -1604,10 +1607,11 @@ public List listGrantsForCatalogRole(String catalogName, String c namespaceGrants.add(grant); break; } - case TABLE_LIKE: + case ICEBERG_TABLE_LIKE: { if (baseEntity.getSubType() == PolarisEntitySubType.TABLE) { - TableIdentifier identifier = TableLikeEntity.of(baseEntity).getTableIdentifier(); + TableIdentifier identifier = + IcebergTableLikeEntity.of(baseEntity).getTableIdentifier(); TableGrant grant = new TableGrant( List.of(identifier.namespace().levels()), @@ -1616,7 +1620,8 @@ public List listGrantsForCatalogRole(String catalogName, String c GrantResource.TypeEnum.TABLE); tableGrants.add(grant); } else { - TableIdentifier identifier = TableLikeEntity.of(baseEntity).getTableIdentifier(); + TableIdentifier identifier = + IcebergTableLikeEntity.of(baseEntity).getTableIdentifier(); ViewGrant grant = new ViewGrant( List.of(identifier.namespace().levels()), @@ -1702,7 +1707,8 @@ private boolean grantPrivilegeOnTableLikeToRole( .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, subType); + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType); if (resolvedPathWrapper == null) { if (subType == PolarisEntitySubType.VIEW) { throw new NotFoundException("View %s not found", identifier); @@ -1740,7 +1746,8 @@ private boolean revokePrivilegeOnTableLikeFromRole( .orElseThrow(() -> new NotFoundException("CatalogRole %s not found", catalogRoleName)); PolarisResolvedPathWrapper resolvedPathWrapper = - resolutionManifest.getResolvedPath(identifier, subType); + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType); if (resolvedPathWrapper == null) { if (subType == PolarisEntitySubType.VIEW) { throw new NotFoundException("View %s not found", identifier); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogPrefixParser.java b/service/common/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java similarity index 97% rename from service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogPrefixParser.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java index 43055fc5ce..7af8fac022 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogPrefixParser.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java @@ -21,7 +21,7 @@ import org.apache.polaris.core.context.RealmContext; /** An extension point for converting Iceberg REST API "prefix" values to Polaris Catalog names. */ -public interface IcebergCatalogPrefixParser { +public interface CatalogPrefixParser { /** * Produces the name of a Polaris catalog from the given Iceberg Catalog REST API "prefix" for the diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/DefaultIcebergCatalogPrefixParser.java b/service/common/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java similarity index 93% rename from service/common/src/main/java/org/apache/polaris/service/catalog/DefaultIcebergCatalogPrefixParser.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java index 42b15bca64..2573c9069f 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/DefaultIcebergCatalogPrefixParser.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java @@ -22,7 +22,7 @@ import org.apache.polaris.core.context.RealmContext; @ApplicationScoped -public class DefaultIcebergCatalogPrefixParser implements IcebergCatalogPrefixParser { +public class DefaultCatalogPrefixParser implements CatalogPrefixParser { @Override public String prefixToCatalogName(RealmContext realm, String prefix) { return prefix; diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java new file mode 100644 index 0000000000..86c4f0bbfe --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.generic; + +import jakarta.ws.rs.core.SecurityContext; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.GenericTableEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.persistence.PolarisEntityManager; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.apache.polaris.service.catalog.io.FileIOFactory; +import org.apache.polaris.service.task.TaskExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GenericTableCatalog { + private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalog.class); + + private final PolarisEntityManager entityManager; + private final CallContext callContext; + private final PolarisResolutionManifestCatalogView resolvedEntityView; + private final CatalogEntity catalogEntity; + private final TaskExecutor taskExecutor; + private final SecurityContext securityContext; + private final String catalogName; + private long catalogId = -1; + private FileIOFactory fileIOFactory; + private PolarisMetaStoreManager metaStoreManager; + + /** + * @param entityManager provides handle to underlying PolarisMetaStoreManager with which to + * perform mutations on entities. + * @param callContext the current CallContext + * @param resolvedEntityView accessor to resolved entity paths that have been pre-vetted to ensure + * this catalog instance only interacts with authorized resolved paths. + * @param taskExecutor Executor we use to register cleanup task handlers + */ + public GenericTableCatalog( + PolarisEntityManager entityManager, + PolarisMetaStoreManager metaStoreManager, + CallContext callContext, + PolarisResolutionManifestCatalogView resolvedEntityView, + SecurityContext securityContext, + TaskExecutor taskExecutor, + FileIOFactory fileIOFactory) { + this.entityManager = entityManager; + this.callContext = callContext; + this.resolvedEntityView = resolvedEntityView; + this.catalogEntity = + CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + this.securityContext = securityContext; + this.taskExecutor = taskExecutor; + this.catalogId = catalogEntity.getId(); + this.catalogName = catalogEntity.getName(); + this.fileIOFactory = fileIOFactory; + this.metaStoreManager = metaStoreManager; + } + + public void createGenericTable( + TableIdentifier tableIdentifier, String format, Map properties) { + PolarisResolvedPathWrapper resolvedParent = + resolvedEntityView.getResolvedPath(tableIdentifier.namespace()); + if (resolvedParent == null) { + // Illegal state because the namespace should've already been in the static resolution set. + throw new IllegalStateException( + String.format( + "Failed to fetch resolved parent for TableIdentifier '%s'", tableIdentifier)); + } + + List catalogPath = resolvedParent.getRawFullPath(); + + // TODO we need to filter by type here? + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.GENERIC_TABLE, PolarisEntitySubType.ANY_SUBTYPE); + GenericTableEntity entity = + GenericTableEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + if (null == entity) { + entity = + new GenericTableEntity.Builder(tableIdentifier, format) + .setCatalogId(this.catalogId) + .setParentNamespace(tableIdentifier.namespace()) + .setParentId(resolvedParent.getRawLeafEntity().getId()) + .setId( + this.metaStoreManager + .generateNewEntityId(this.callContext.getPolarisCallContext()) + .getId()) + .setProperties(properties) + .setCreateTimestamp(System.currentTimeMillis()) + .build(); + } else { + throw new AlreadyExistsException( + "Iceberg table, view, or generic table already exists: %s", tableIdentifier); + } + + EntityResult res = + this.metaStoreManager.createEntityIfNotExists( + this.callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + entity); + if (!res.isSuccess()) { + switch (res.getReturnStatus()) { + case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS: + throw new AlreadyExistsException( + "Iceberg table, view, or generic table already exists: %s", tableIdentifier); + + default: + throw new IllegalStateException( + String.format( + "Unknown error status for identifier %s: %s with extraInfo: %s", + tableIdentifier, res.getReturnStatus(), res.getExtraInformation())); + } + } + PolarisEntity resultEntity = PolarisEntity.of(res); + LOGGER.debug( + "Created GenericTable entity {} with TableIdentifier {}", resultEntity, tableIdentifier); + } + + public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) { + // TODO we need to filter by type here? + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.GENERIC_TABLE, PolarisEntitySubType.ANY_SUBTYPE); + GenericTableEntity entity = + GenericTableEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + if (null == entity) { + throw new NoSuchTableException("Table does not exist: %s", tableIdentifier); + } else { + return entity; + } + } +} diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java similarity index 93% rename from service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 582394c373..00469cc491 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.catalog; +package org.apache.polaris.service.catalog.iceberg; import static org.apache.polaris.service.exception.IcebergExceptionMapper.isStorageProviderRetryableException; @@ -83,13 +83,13 @@ import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.NamespaceEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; @@ -108,6 +108,7 @@ import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageIntegration; import org.apache.polaris.core.storage.StorageLocation; +import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.FileIOUtil; import org.apache.polaris.service.task.TaskExecutor; @@ -117,9 +118,9 @@ import org.slf4j.LoggerFactory; /** Defines the relationship between PolarisEntities and Iceberg's business logic. */ -public class BasePolarisCatalog extends BaseMetastoreViewCatalog +public class IcebergCatalog extends BaseMetastoreViewCatalog implements SupportsNamespaces, SupportsNotifications, Closeable, SupportsCredentialDelegation { - private static final Logger LOGGER = LoggerFactory.getLogger(BasePolarisCatalog.class); + private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalog.class); private static final Joiner SLASH = Joiner.on("/"); @@ -131,7 +132,7 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog // Config key for initializing a default "catalogFileIO" that is available either via getIo() // or for any TableOperations/ViewOperations instantiated, via ops.io() before entity-specific // FileIO initialization is triggered for any such operations. - // Typically this should only be used in test scenarios where a BasePolarisCatalog instance + // Typically this should only be used in test scenarios where a PolarisIcebergCatalog instance // is used for both the "client-side" and "server-side" logic instead of being access through // a REST layer. static final String INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST = @@ -177,7 +178,7 @@ public class BasePolarisCatalog extends BaseMetastoreViewCatalog * this catalog instance only interacts with authorized resolved paths. * @param taskExecutor Executor we use to register cleanup task handlers */ - public BasePolarisCatalog( + public IcebergCatalog( PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, CallContext callContext, @@ -337,12 +338,12 @@ public Table registerTable(TableIdentifier identifier, String metadataFileLocati @Override public TableBuilder buildTable(TableIdentifier identifier, Schema schema) { - return new BasePolarisCatalogTableBuilder(identifier, schema); + return new PolarisIcebergCatalogTableBuilder(identifier, schema); } @Override public ViewBuilder buildView(TableIdentifier identifier) { - return new BasePolarisCatalogViewBuilder(identifier); + return new PolarisIcebergCatalogViewBuilder(identifier); } @Override @@ -381,17 +382,19 @@ private Set getLocationsAllowedToBeAccessed(TableMetadata tableMetadata) locations.add(tableMetadata.location()); if (tableMetadata .properties() - .containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { + .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)) { locations.add( - tableMetadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); + tableMetadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); } if (tableMetadata .properties() - .containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) { + .containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)) { locations.add( tableMetadata .properties() - .get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); } return locations; } @@ -565,7 +568,7 @@ private String resolveNamespaceLocation(Namespace namespace, Map .reversed() .stream() .map(entity -> baseLocation(callContext, entity)) - .map(BasePolarisCatalog::stripLeadingTrailingSlash) + .map(IcebergCatalog::stripLeadingTrailingSlash) .collect(Collectors.joining("/")); } @@ -874,7 +877,8 @@ public String transformTableLikeLocation(String specifiedTableLikeLocation) { private @Nonnull Optional findStorageInfo(TableIdentifier tableIdentifier) { PolarisResolvedPathWrapper resolvedTableEntities = - resolvedEntityView.getResolvedPath(tableIdentifier, PolarisEntitySubType.TABLE); + resolvedEntityView.getResolvedPath( + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE); PolarisResolvedPathWrapper resolvedStorageEntity = resolvedTableEntities == null @@ -890,7 +894,8 @@ public String transformTableLikeLocation(String specifiedTableLikeLocation) { */ private void validateLocationForTableLike(TableIdentifier identifier, String location) { PolarisResolvedPathWrapper resolvedStorageEntity = - resolvedEntityView.getResolvedPath(identifier, PolarisEntitySubType.ANY_SUBTYPE); + resolvedEntityView.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); if (resolvedStorageEntity == null) { resolvedStorageEntity = resolvedEntityView.getResolvedPath(identifier.namespace()); } @@ -1063,7 +1068,7 @@ private void validateNoLocationOverlap( parentPath.stream() .map(PolarisEntity::toCore) .collect(Collectors.toList()), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); if (!siblingTablesResult.isSuccess()) { throw new IllegalStateException( @@ -1098,7 +1103,8 @@ private void validateNoLocationOverlap( tbl -> resolutionManifest.addPath( new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(tbl), PolarisEntityType.TABLE_LIKE), + PolarisCatalogHelpers.tableIdentifierToList(tbl), + PolarisEntityType.ICEBERG_TABLE_LIKE), tbl)); siblingNamespaces.forEach( ns -> @@ -1119,7 +1125,7 @@ private void validateNoLocationOverlap( tbl -> { PolarisResolvedPathWrapper resolveTablePath = resolutionManifest.getResolvedPath(tbl); - return TableLikeEntity.of(resolveTablePath.getRawLeafEntity()) + return IcebergTableLikeEntity.of(resolveTablePath.getRawLeafEntity()) .getBaseLocation(); }), siblingNamespaces.stream() @@ -1144,10 +1150,10 @@ private void validateNoLocationOverlap( }); } - private class BasePolarisCatalogTableBuilder + private class PolarisIcebergCatalogTableBuilder extends BaseMetastoreViewCatalog.BaseMetastoreViewCatalogTableBuilder { - public BasePolarisCatalogTableBuilder(TableIdentifier identifier, Schema schema) { + public PolarisIcebergCatalogTableBuilder(TableIdentifier identifier, Schema schema) { super(identifier, schema); } @@ -1157,13 +1163,12 @@ public TableBuilder withLocation(String newLocation) { } } - private class BasePolarisCatalogViewBuilder extends BaseMetastoreViewCatalog.BaseViewBuilder { + private class PolarisIcebergCatalogViewBuilder extends BaseMetastoreViewCatalog.BaseViewBuilder { - public BasePolarisCatalogViewBuilder(TableIdentifier identifier) { + public PolarisIcebergCatalogViewBuilder(TableIdentifier identifier) { super(identifier); withProperties( - PropertyUtil.propertiesWithPrefix( - BasePolarisCatalog.this.properties(), "table-default.")); + PropertyUtil.propertiesWithPrefix(IcebergCatalog.this.properties(), "table-default.")); } @Override @@ -1191,11 +1196,11 @@ public void doRefresh() { // table entity instead of the statically-resolved authz resolution set. PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - TableLikeEntity entity = null; + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE); + IcebergTableLikeEntity entity = null; if (resolvedEntities != null) { - entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + entity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); if (!tableIdentifier.equals(entity.getTableIdentifier())) { LOGGER .atError() @@ -1245,7 +1250,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { PolarisResolvedPathWrapper resolvedTableEntities = resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE); // Fetch credentials for the resolved entity. The entity could be the table itself (if it has // already been stored and credentials have been configured directly) or it could be the @@ -1273,23 +1278,29 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { if (base == null || !metadata.location().equals(base.location()) || !Objects.equal( - base.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), - metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY))) { + base.properties().get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY), + metadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY))) { // If location is changing then we must validate that the requested location is valid // for the storage configuration inherited under this entity's path. Set dataLocations = new HashSet<>(); dataLocations.add(metadata.location()); - if (metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) + if (metadata.properties().get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) != null) { dataLocations.add( - metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); + metadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY)); } - if (metadata.properties().get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY) + if (metadata + .properties() + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY) != null) { dataLocations.add( metadata .properties() - .get(TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); + .get(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY)); } validateLocationsForTableLike(tableIdentifier, dataLocations, resolvedStorageEntity); // also validate that the table location doesn't overlap an existing table @@ -1311,7 +1322,8 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { String oldLocation = base == null ? null : base.metadataFileLocation(); PolarisResolvedPathWrapper resolvedView = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, PolarisEntitySubType.VIEW); + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW); if (resolvedView != null) { throw new AlreadyExistsException("View with same name already exists: %s", tableIdentifier); } @@ -1324,14 +1336,15 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { // persistence-layer commit). PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntitySubType.TABLE); - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE); + IcebergTableLikeEntity entity = + IcebergTableLikeEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); String existingLocation; if (null == entity) { existingLocation = null; entity = - new TableLikeEntity.Builder(tableIdentifier, newLocation) + new IcebergTableLikeEntity.Builder(tableIdentifier, newLocation) .setCatalogId(getCatalogId()) .setSubType(PolarisEntitySubType.TABLE) .setBaseLocation(metadata.location()) @@ -1341,7 +1354,7 @@ public void doCommit(TableMetadata base, TableMetadata metadata) { } else { existingLocation = entity.getMetadataLocation(); entity = - new TableLikeEntity.Builder(entity) + new IcebergTableLikeEntity.Builder(entity) .setBaseLocation(metadata.location()) .setMetadataLocation(newLocation) .build(); @@ -1420,11 +1433,12 @@ private class BasePolarisViewOperations extends BaseViewOperations { @Override public void doRefresh() { PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.VIEW); - TableLikeEntity entity = null; + resolvedEntityView.getPassthroughResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW); + IcebergTableLikeEntity entity = null; if (resolvedEntities != null) { - entity = TableLikeEntity.of(resolvedEntities.getRawLeafEntity()); + entity = IcebergTableLikeEntity.of(resolvedEntities.getRawLeafEntity()); if (!identifier.equals(entity.getTableIdentifier())) { LOGGER .atError() @@ -1474,13 +1488,15 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) { } PolarisResolvedPathWrapper resolvedTable = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.TABLE); + resolvedEntityView.getPassthroughResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.TABLE); if (resolvedTable != null) { throw new AlreadyExistsException("Table with same name already exists: %s", identifier); } PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(identifier, PolarisEntitySubType.VIEW); + resolvedEntityView.getPassthroughResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, PolarisEntitySubType.VIEW); // Fetch credentials for the resolved entity. The entity could be the view itself (if it has // already been stored and credentials have been configured directly) or it could be the @@ -1519,13 +1535,14 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) { String newLocation = writeNewMetadataIfRequired(metadata); String oldLocation = base == null ? null : currentMetadataLocation(); - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + IcebergTableLikeEntity entity = + IcebergTableLikeEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); String existingLocation; if (null == entity) { existingLocation = null; entity = - new TableLikeEntity.Builder(identifier, newLocation) + new IcebergTableLikeEntity.Builder(identifier, newLocation) .setCatalogId(getCatalogId()) .setSubType(PolarisEntitySubType.VIEW) .setId( @@ -1533,7 +1550,8 @@ public void doCommit(ViewMetadata base, ViewMetadata metadata) { .build(); } else { existingLocation = entity.getMetadataLocation(); - entity = new TableLikeEntity.Builder(entity).setMetadataLocation(newLocation).build(); + entity = + new IcebergTableLikeEntity.Builder(entity).setMetadataLocation(newLocation).build(); } if (!Objects.equal(existingLocation, oldLocation)) { if (null == base) { @@ -1618,7 +1636,8 @@ long getCatalogId() { private void renameTableLike( PolarisEntitySubType subType, TableIdentifier from, TableIdentifier to) { LOGGER.debug("Renaming tableLike from {} to {}", from, to); - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(from, subType); + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getResolvedPath(from, PolarisEntityType.ICEBERG_TABLE_LIKE, subType); if (resolvedEntities == null) { if (subType == PolarisEntitySubType.VIEW) { throw new NoSuchViewException("Cannot rename %s to %s. View does not exist", from, to); @@ -1628,7 +1647,7 @@ private void renameTableLike( } List catalogPath = resolvedEntities.getRawParentPath(); PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); - final TableLikeEntity toEntity; + final IcebergTableLikeEntity toEntity; List newCatalogPath = null; if (!from.namespace().equals(to.namespace())) { PolarisResolvedPathWrapper resolvedNewParentEntities = @@ -1641,14 +1660,14 @@ private void renameTableLike( // the "to" table has a new parent and a new name / namespace path toEntity = - new TableLikeEntity.Builder(TableLikeEntity.of(leafEntity)) + new IcebergTableLikeEntity.Builder(IcebergTableLikeEntity.of(leafEntity)) .setTableIdentifier(to) .setParentId(resolvedNewParentEntities.getResolvedLeafEntity().getEntity().getId()) .build(); } else { // only the name of the entity is changed toEntity = - new TableLikeEntity.Builder(TableLikeEntity.of(leafEntity)) + new IcebergTableLikeEntity.Builder(IcebergTableLikeEntity.of(leafEntity)) .setTableIdentifier(to) .build(); } @@ -1708,7 +1727,8 @@ private void renameTableLike( "Unknown error status " + returnedEntityResult.getReturnStatus()); } } else { - TableLikeEntity returnedEntity = TableLikeEntity.of(returnedEntityResult.getEntity()); + IcebergTableLikeEntity returnedEntity = + IcebergTableLikeEntity.of(returnedEntityResult.getEntity()); if (!toEntity.getTableIdentifier().equals(returnedEntity.getTableIdentifier())) { // As long as there are older deployments which don't support the atomic update of the // internalProperties during rename, we can log and then patch it up explicitly @@ -1722,7 +1742,7 @@ private void renameTableLike( .updateEntityPropertiesIfNotChanged( getCurrentPolarisContext(), PolarisEntity.toCoreList(newCatalogPath), - new TableLikeEntity.Builder(returnedEntity).setTableIdentifier(to).build()); + new IcebergTableLikeEntity.Builder(returnedEntity).setTableIdentifier(to).build()); } } } @@ -1747,7 +1767,7 @@ private void createTableLike(TableIdentifier identifier, PolarisEntity entity) { private void createTableLike( TableIdentifier identifier, PolarisEntity entity, PolarisResolvedPathWrapper resolvedParent) { // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = TableLikeEntity.of(entity).getMetadataLocation(); + String metadataLocation = IcebergTableLikeEntity.of(entity).getMetadataLocation(); validateLocationForTableLike(identifier, metadataLocation, resolvedParent); List catalogPath = resolvedParent.getRawFullPath(); @@ -1772,8 +1792,8 @@ private void createTableLike( throw new NotFoundException("Parent path does not exist for %s", identifier); case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS: - throw new AlreadyExistsException("Table or View already exists: %s", identifier); - + throw new AlreadyExistsException( + "Iceberg table, view, or generic table already exists: %s", identifier); default: throw new IllegalStateException( String.format( @@ -1787,7 +1807,7 @@ private void createTableLike( private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(identifier, entity.getSubType()); + resolvedEntityView.getResolvedPath(identifier, entity.getType(), entity.getSubType()); if (resolvedEntities == null) { // Illegal state because the identifier should've already been in the static resolution set. throw new IllegalStateException( @@ -1795,7 +1815,7 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { } // Make sure the metadata file is valid for our allowed locations. - String metadataLocation = TableLikeEntity.of(entity).getMetadataLocation(); + String metadataLocation = IcebergTableLikeEntity.of(entity).getMetadataLocation(); validateLocationForTableLike(identifier, metadataLocation, resolvedEntities); List catalogPath = resolvedEntities.getRawParentPath(); @@ -1830,7 +1850,8 @@ private void updateTableLike(TableIdentifier identifier, PolarisEntity entity) { Map storageProperties, boolean purge) { PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getResolvedPath(identifier, subType); + resolvedEntityView.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType); if (resolvedEntities == null) { // TODO: Error? return new DropEntityResult(BaseResult.ReturnStatus.ENTITY_NOT_FOUND, null); @@ -1874,7 +1895,8 @@ private boolean sendNotificationForTableLike( LOGGER.debug( "Handling notification request {} for tableIdentifier {}", request, tableIdentifier); PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath(tableIdentifier, subType); + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType); NotificationType notificationType = request.getNotificationType(); @@ -1936,15 +1958,16 @@ private boolean sendNotificationForTableLike( PolarisResolvedPathWrapper resolvedParent = resolvedEntityView.getPassthroughResolvedPath(ns); - TableLikeEntity entity = - TableLikeEntity.of(resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + IcebergTableLikeEntity entity = + IcebergTableLikeEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); String existingLocation; String newLocation = transformTableLikeLocation(request.getPayload().getMetadataLocation()); if (null == entity) { existingLocation = null; entity = - new TableLikeEntity.Builder(tableIdentifier, newLocation) + new IcebergTableLikeEntity.Builder(tableIdentifier, newLocation) .setCatalogId(getCatalogId()) .setSubType(PolarisEntitySubType.TABLE) .setId( @@ -1962,7 +1985,7 @@ private boolean sendNotificationForTableLike( } existingLocation = entity.getMetadataLocation(); entity = - new TableLikeEntity.Builder(entity) + new IcebergTableLikeEntity.Builder(entity) .setMetadataLocation(newLocation) .setLastNotificationTimestamp(request.getPayload().getTimestamp()) .build(); @@ -2038,7 +2061,7 @@ private List listTableLike(PolarisEntitySubType subType, Namesp .listEntities( getCurrentPolarisContext(), PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, subType) .getEntities()); return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); @@ -2052,8 +2075,8 @@ private List listTableLike(PolarisEntitySubType subType, Namesp * @return FileIO object */ private FileIO loadFileIO(String ioImpl, Map properties) { - TableLikeEntity tableLikeEntity = TableLikeEntity.of(catalogEntity); - TableIdentifier identifier = tableLikeEntity.getTableIdentifier(); + IcebergTableLikeEntity icebergTableLikeEntity = IcebergTableLikeEntity.of(catalogEntity); + TableIdentifier identifier = icebergTableLikeEntity.getTableIdentifier(); Set locations = Set.of(catalogEntity.getDefaultBaseLocation()); ResolvedPolarisEntity resolvedCatalogEntity = new ResolvedPolarisEntity(catalogEntity, List.of(), List.of()); @@ -2066,9 +2089,9 @@ private FileIO loadFileIO(String ioImpl, Map properties) { private void blockedUserSpecifiedWriteLocation(Map properties) { if (properties != null - && (properties.containsKey(TableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) + && (properties.containsKey(IcebergTableLikeEntity.USER_SPECIFIED_WRITE_DATA_LOCATION_KEY) || properties.containsKey( - TableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY))) { + IcebergTableLikeEntity.USER_SPECIFIED_WRITE_METADATA_LOCATION_KEY))) { throw new ForbiddenException( "Delegate access to table with user-specified write location is temporarily not supported."); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java similarity index 97% rename from service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 0ce0c2db7b..be2574267a 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.catalog; +package org.apache.polaris.service.catalog.iceberg; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; @@ -68,6 +68,8 @@ import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.Resolver; import org.apache.polaris.core.persistence.resolver.ResolverStatus; +import org.apache.polaris.service.catalog.AccessDelegationMode; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApiService; import org.apache.polaris.service.context.CallContextCatalogFactory; @@ -126,7 +128,7 @@ public class IcebergCatalogAdapter private final PolarisEntityManager entityManager; private final PolarisMetaStoreManager metaStoreManager; private final PolarisAuthorizer polarisAuthorizer; - private final IcebergCatalogPrefixParser prefixParser; + private final CatalogPrefixParser prefixParser; @Inject public IcebergCatalogAdapter( @@ -139,7 +141,7 @@ public IcebergCatalogAdapter( PolarisConfigurationStore configurationStore, PolarisDiagnostics diagnostics, PolarisAuthorizer polarisAuthorizer, - IcebergCatalogPrefixParser prefixParser) { + CatalogPrefixParser prefixParser) { this.realmContext = realmContext; this.callContext = callContext; this.catalogFactory = catalogFactory; @@ -159,9 +161,9 @@ public IcebergCatalogAdapter( private Response withCatalog( SecurityContext securityContext, String prefix, - Function action) { + Function action) { String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); - try (PolarisCatalogHandlerWrapper wrapper = + try (IcebergCatalogHandlerWrapper wrapper = newHandlerWrapper(realmContext, securityContext, catalogName)) { return action.apply(wrapper); } catch (RuntimeException e) { @@ -173,7 +175,7 @@ private Response withCatalog( } } - private PolarisCatalogHandlerWrapper newHandlerWrapper( + private IcebergCatalogHandlerWrapper newHandlerWrapper( RealmContext realmContext, SecurityContext securityContext, String catalogName) { AuthenticatedPolarisPrincipal authenticatedPrincipal = (AuthenticatedPolarisPrincipal) securityContext.getUserPrincipal(); @@ -181,7 +183,7 @@ private PolarisCatalogHandlerWrapper newHandlerWrapper( throw new NotAuthorizedException("Failed to find authenticatedPrincipal in SecurityContext"); } - return new PolarisCatalogHandlerWrapper( + return new IcebergCatalogHandlerWrapper( callContext, entityManager, metaStoreManager, @@ -415,7 +417,7 @@ public Response updateTable( Namespace ns = decodeNamespace(namespace); TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); - if (PolarisCatalogHandlerWrapper.isCreate(commitTableRequest)) { + if (IcebergCatalogHandlerWrapper.isCreate(commitTableRequest)) { return Response.ok( newHandlerWrapper(realmContext, securityContext, prefix) .updateTableForStagedCreate(tableIdentifier, commitTableRequest)) diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java similarity index 96% rename from service/common/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java index 837fb509f7..c774525c84 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/PolarisCatalogHandlerWrapper.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerWrapper.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.catalog; +package org.apache.polaris.service.catalog.iceberg; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; @@ -92,6 +92,7 @@ import org.apache.polaris.core.persistence.resolver.ResolverPath; import org.apache.polaris.core.persistence.resolver.ResolverStatus; import org.apache.polaris.core.storage.PolarisStorageActions; +import org.apache.polaris.service.catalog.SupportsNotifications; import org.apache.polaris.service.context.CallContextCatalogFactory; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; @@ -101,7 +102,7 @@ * Authorization-aware adapter between REST stubs and shared Iceberg SDK CatalogHandlers. * *

We must make authorization decisions based on entity resolution at this layer instead of the - * underlying BasePolarisCatalog layer, because this REST-adjacent layer captures intent of + * underlying PolarisIcebergCatalog layer, because this REST-adjacent layer captures intent of * different REST calls that share underlying catalog calls (e.g. updateTable will call loadTable * under the hood), and some features of the REST API aren't expressed at all in the underlying * Catalog interfaces (e.g. credential-vending in createTable/loadTable). @@ -112,8 +113,8 @@ * model objects used in this layer to still benefit from the shared implementation of * authorization-aware catalog protocols. */ -public class PolarisCatalogHandlerWrapper implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(PolarisCatalogHandlerWrapper.class); +public class IcebergCatalogHandlerWrapper implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(IcebergCatalogHandlerWrapper.class); private final CallContext callContext; private final PolarisEntityManager entityManager; @@ -133,7 +134,7 @@ public class PolarisCatalogHandlerWrapper implements AutoCloseable { private SupportsNamespaces namespaceCatalog = null; private ViewCatalog viewCatalog = null; - public PolarisCatalogHandlerWrapper( + public IcebergCatalogHandlerWrapper( CallContext callContext, PolarisEntityManager entityManager, PolarisMetaStoreManager metaStoreManager, @@ -218,7 +219,7 @@ private void authorizeBasicNamespaceOperationOrThrow( resolutionManifest.addPassthroughPath( new ResolverPath( PolarisCatalogHelpers.tableIdentifierToList(id), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, true /* optional */), id); } @@ -290,7 +291,7 @@ private void authorizeCreateTableLikeUnderNamespaceOperationOrThrow( resolutionManifest.addPassthroughPath( new ResolverPath( PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, true /* optional */), identifier); resolutionManifest.resolveAll(); @@ -317,12 +318,13 @@ private void authorizeBasicTableLikeOperationOrThrow( resolutionManifest.addPassthroughPath( new ResolverPath( PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, true /* optional */), identifier); resolutionManifest.resolveAll(); PolarisResolvedPathWrapper target = - resolutionManifest.getResolvedPath(identifier, subType, true); + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType, true); if (target == null) { if (subType == PolarisEntitySubType.TABLE) { throw new NoSuchTableException("Table does not exist: %s", identifier); @@ -351,7 +353,7 @@ private void authorizeCollectionOfTableLikeOperationOrThrow( resolutionManifest.addPassthroughPath( new ResolverPath( PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), + PolarisEntityType.ICEBERG_TABLE_LIKE), identifier)); ResolverStatus status = resolutionManifest.resolveAll(); @@ -374,7 +376,8 @@ private void authorizeCollectionOfTableLikeOperationOrThrow( .map( identifier -> Optional.ofNullable( - resolutionManifest.getResolvedPath(identifier, subType, true)) + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.ICEBERG_TABLE_LIKE, subType, true)) .orElseThrow( () -> subType == PolarisEntitySubType.TABLE @@ -403,7 +406,7 @@ private void authorizeRenameTableLikeOperationOrThrow( // Add src, dstParent, and dst(optional) resolutionManifest.addPath( new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(src), PolarisEntityType.TABLE_LIKE), + PolarisCatalogHelpers.tableIdentifierToList(src), PolarisEntityType.ICEBERG_TABLE_LIKE), src); resolutionManifest.addPath( new ResolverPath(Arrays.asList(dst.namespace().levels()), PolarisEntityType.NAMESPACE), @@ -411,14 +414,16 @@ private void authorizeRenameTableLikeOperationOrThrow( resolutionManifest.addPath( new ResolverPath( PolarisCatalogHelpers.tableIdentifierToList(dst), - PolarisEntityType.TABLE_LIKE, + PolarisEntityType.ICEBERG_TABLE_LIKE, true /* optional */), dst); ResolverStatus status = resolutionManifest.resolveAll(); if (status.getStatus() == ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED && status.getFailedToResolvePath().getLastEntityType() == PolarisEntityType.NAMESPACE) { throw new NoSuchNamespaceException("Namespace does not exist: %s", dst.namespace()); - } else if (resolutionManifest.getResolvedPath(src, subType) == null) { + } else if (resolutionManifest.getResolvedPath( + src, PolarisEntityType.ICEBERG_TABLE_LIKE, subType) + == null) { if (subType == PolarisEntitySubType.TABLE) { throw new NoSuchTableException("Table does not exist: %s", src); } else { @@ -440,7 +445,9 @@ private void authorizeRenameTableLikeOperationOrThrow( throw new AlreadyExistsException("Cannot rename %s to %s. View already exists", src, dst); } - PolarisResolvedPathWrapper target = resolutionManifest.getResolvedPath(src, subType, true); + PolarisResolvedPathWrapper target = + resolutionManifest.getResolvedPath( + src, PolarisEntityType.ICEBERG_TABLE_LIKE, subType, true); PolarisResolvedPathWrapper secondary = resolutionManifest.getResolvedPath(dst.namespace(), true); authorizer.authorizeOrThrow( @@ -470,7 +477,7 @@ public CreateNamespaceResponse createNamespace(CreateNamespaceRequest request) { } authorizeCreateNamespaceUnderNamespaceOperationOrThrow(op, namespace); - if (namespaceCatalog instanceof BasePolarisCatalog) { + if (namespaceCatalog instanceof IcebergCatalog) { // Note: The CatalogHandlers' default implementation will non-atomically create the // namespace and then fetch its properties using loadNamespaceMetadata for the response. // However, the latest namespace metadata technically isn't the same authorized instance, @@ -673,9 +680,8 @@ private TableMetadata stageTableCreateHelper(Namespace namespace, CreateTableReq if (request.location() != null) { // Even if the request provides a location, run it through the catalog's TableBuilder // to inherit any override behaviors if applicable. - if (baseCatalog instanceof BasePolarisCatalog) { - location = - ((BasePolarisCatalog) baseCatalog).transformTableLikeLocation(request.location()); + if (baseCatalog instanceof IcebergCatalog) { + location = ((IcebergCatalog) baseCatalog).transformTableLikeLocation(request.location()); } else { location = request.location(); } @@ -772,7 +778,7 @@ public boolean sendNotification(TableIdentifier identifier, NotificationRequest PolarisAuthorizableOperation op = PolarisAuthorizableOperation.SEND_NOTIFICATIONS; // For now, just require the full set of privileges on the base Catalog entity, which we can - // also express just as the "root" Namespace for purposes of the BasePolarisCatalog being + // also express just as the "root" Namespace for purposes of the PolarisIcebergCatalog being // able to fetch Namespace.empty() as path key. List extraPassthroughTableLikes = List.of(identifier); List extraPassthroughNamespaces = new ArrayList<>(); @@ -896,11 +902,11 @@ private UpdateTableRequest applyUpdateFilters(UpdateTableRequest request) { request.updates().stream() .map( update -> { - if (baseCatalog instanceof BasePolarisCatalog + if (baseCatalog instanceof IcebergCatalog && update instanceof MetadataUpdate.SetLocation) { String requestedLocation = ((MetadataUpdate.SetLocation) update).location(); String filteredLocation = - ((BasePolarisCatalog) baseCatalog) + ((IcebergCatalog) baseCatalog) .transformTableLikeLocation(requestedLocation); return new MetadataUpdate.SetLocation(filteredLocation); } else { @@ -1019,7 +1025,7 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) throw new BadRequestException("Cannot update table on external catalogs."); } - if (!(baseCatalog instanceof BasePolarisCatalog)) { + if (!(baseCatalog instanceof IcebergCatalog)) { throw new BadRequestException( "Unsupported operation: commitTransaction with baseCatalog type: %s", baseCatalog.getClass().getName()); @@ -1030,7 +1036,7 @@ public void commitTransaction(CommitTransactionRequest commitTransactionRequest) // validations. TransactionWorkspaceMetaStoreManager transactionMetaStoreManager = new TransactionWorkspaceMetaStoreManager(metaStoreManager); - ((BasePolarisCatalog) baseCatalog).setMetaStoreManager(transactionMetaStoreManager); + ((IcebergCatalog) baseCatalog).setMetaStoreManager(transactionMetaStoreManager); commitTransactionRequest.tableChanges().stream() .forEach( diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java similarity index 96% rename from service/common/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java rename to service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java index 554358765e..06ca7fbde6 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/SupportsCredentialDelegation.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/SupportsCredentialDelegation.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.catalog; +package org.apache.polaris.service.catalog.iceberg; import java.util.Map; import java.util.Set; diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java index f70c594ddf..481033dad0 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java @@ -31,7 +31,7 @@ import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.storage.PolarisCredentialVendor; import org.apache.polaris.core.storage.PolarisStorageActions; -import org.apache.polaris.service.catalog.BasePolarisCatalog; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,7 +69,7 @@ public static Optional findStorageInfoFromHierarchy( *

Use cases: * *

    - *
  • In {@link BasePolarisCatalog}, subscoped credentials are generated or refreshed when the + *
  • In {@link IcebergCatalog}, subscoped credentials are generated or refreshed when the * client sends a loadTable request to vend credentials. *
  • In {@link DefaultFileIOFactory}, subscoped credentials are obtained to access the storage * and read/write metadata JSON files. diff --git a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java index c341ea6fa4..2332e9a00d 100644 --- a/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/context/PolarisCallContextCatalogFactory.java @@ -34,7 +34,7 @@ import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; -import org.apache.polaris.service.catalog.BasePolarisCatalog; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.task.TaskExecutor; @@ -83,8 +83,8 @@ public Catalog createCallContextCatalog( PolarisEntityManager entityManager = entityManagerFactory.getOrCreateEntityManager(context.getRealmContext()); - BasePolarisCatalog catalogInstance = - new BasePolarisCatalog( + IcebergCatalog catalogInstance = + new IcebergCatalog( entityManager, metaStoreManagerFactory.getOrCreateMetaStoreManager(context.getRealmContext()), context, diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java b/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java index b656ca5300..35ba9440ec 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TableCleanupTaskHandler.java @@ -33,10 +33,10 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.AsyncTaskType; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntity; import org.apache.polaris.core.entity.PolarisEntityType; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; @@ -71,7 +71,7 @@ public boolean canHandleTask(TaskEntity task) { private boolean taskEntityIsTable(TaskEntity task) { PolarisEntity entity = PolarisEntity.of((task.readData(PolarisBaseEntity.class))); - return entity.getType().equals(PolarisEntityType.TABLE_LIKE); + return entity.getType().equals(PolarisEntityType.ICEBERG_TABLE_LIKE); } @Override @@ -79,7 +79,7 @@ public boolean handleTask(TaskEntity cleanupTask, CallContext callContext) { PolarisBaseEntity entity = cleanupTask.readData(PolarisBaseEntity.class); PolarisMetaStoreManager metaStoreManager = metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()); - TableLikeEntity tableEntity = TableLikeEntity.of(entity); + IcebergTableLikeEntity tableEntity = IcebergTableLikeEntity.of(entity); PolarisCallContext polarisCallContext = callContext.getPolarisCallContext(); LOGGER .atInfo() @@ -153,7 +153,7 @@ private Stream getManifestTaskStream( TaskEntity cleanupTask, TableMetadata tableMetadata, FileIO fileIO, - TableLikeEntity tableEntity, + IcebergTableLikeEntity tableEntity, PolarisMetaStoreManager metaStoreManager, PolarisCallContext polarisCallContext) { // read the manifest list for each snapshot. dedupe the manifest files and schedule a @@ -204,7 +204,7 @@ private Stream getMetadataTaskStream( TaskEntity cleanupTask, TableMetadata tableMetadata, FileIO fileIO, - TableLikeEntity tableEntity, + IcebergTableLikeEntity tableEntity, PolarisMetaStoreManager metaStoreManager, PolarisCallContext polarisCallContext) { int batchSize = diff --git a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index 679d48060b..b87c062718 100644 --- a/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/service/common/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -29,8 +29,8 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.io.FileIO; import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.IcebergTableLikeEntity; import org.apache.polaris.core.entity.PolarisTaskConstants; -import org.apache.polaris.core.entity.TableLikeEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; @@ -51,7 +51,7 @@ public FileIO apply(TaskEntity task, CallContext callContext) { Map internalProperties = task.getInternalPropertiesAsMap(); Map properties = new HashMap<>(internalProperties); - TableLikeEntity tableEntity = TableLikeEntity.of(task); + IcebergTableLikeEntity tableEntity = IcebergTableLikeEntity.of(task); TableIdentifier identifier = tableEntity.getTableIdentifier(); String location = properties.get(PolarisTaskConstants.STORAGE_LOCATION); Set locations = Set.of(location); diff --git a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java index 760185a1de..9a070273b8 100644 --- a/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java +++ b/service/common/src/test/java/org/apache/polaris/service/catalog/io/FileIOFactoryTest.java @@ -47,8 +47,8 @@ import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.entity.*; import org.apache.polaris.service.TestServices; -import org.apache.polaris.service.catalog.BasePolarisCatalog; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.task.TaskFileIOSupplier; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; @@ -157,7 +157,7 @@ public void after() {} @Test public void testLoadFileIOForTableLike() { - BasePolarisCatalog catalog = createCatalog(testServices); + IcebergCatalog catalog = createCatalog(testServices); catalog.createNamespace(NS); catalog.createTable(TABLE, SCHEMA); @@ -175,7 +175,7 @@ public void testLoadFileIOForTableLike() { @Test public void testLoadFileIOForCleanupTask() { - BasePolarisCatalog catalog = createCatalog(testServices); + IcebergCatalog catalog = createCatalog(testServices); catalog.createNamespace(NS); catalog.createTable(TABLE, SCHEMA); catalog.dropTable(TABLE, true); @@ -208,7 +208,7 @@ public void testLoadFileIOForCleanupTask() { Mockito.any()); } - BasePolarisCatalog createCatalog(TestServices services) { + IcebergCatalog createCatalog(TestServices services) { String storageLocation = "s3://my-bucket/path/to/data"; AwsStorageConfigInfo awsStorageConfigInfo = AwsStorageConfigInfo.builder() @@ -236,8 +236,8 @@ BasePolarisCatalog createCatalog(TestServices services) { services.entityManagerFactory().getOrCreateEntityManager(realmContext), services.securityContext(), CATALOG_NAME); - BasePolarisCatalog polarisCatalog = - new BasePolarisCatalog( + IcebergCatalog polarisCatalog = + new IcebergCatalog( services.entityManagerFactory().getOrCreateEntityManager(realmContext), services.metaStoreManagerFactory().getOrCreateMetaStoreManager(realmContext), callContext, diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 0f63106ac2..f14b27a391 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -44,10 +44,10 @@ import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; -import org.apache.polaris.service.catalog.DefaultIcebergCatalogPrefixParser; -import org.apache.polaris.service.catalog.IcebergCatalogAdapter; +import org.apache.polaris.service.catalog.DefaultCatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApiService; +import org.apache.polaris.service.catalog.iceberg.IcebergCatalogAdapter; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.io.MeasuredFileIOFactory; import org.apache.polaris.service.config.DefaultConfigurationStore; @@ -181,7 +181,7 @@ public Map contextVariables() { configurationStore, polarisDiagnostics, authorizer, - new DefaultIcebergCatalogPrefixParser()); + new DefaultCatalogPrefixParser()); IcebergRestCatalogApi restApi = new IcebergRestCatalogApi(service); diff --git a/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java b/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java index 4b7de659fe..8c98e910b1 100644 --- a/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java +++ b/service/common/src/testFixtures/java/org/apache/polaris/service/catalog/PolarisPassthroughResolutionView.java @@ -83,23 +83,22 @@ public PolarisResolvedPathWrapper getResolvedPath(Object key) { } @Override - public PolarisResolvedPathWrapper getResolvedPath(Object key, PolarisEntitySubType subType) { + public PolarisResolvedPathWrapper getResolvedPath( + Object key, PolarisEntityType entityType, PolarisEntitySubType subType) { PolarisResolutionManifest manifest = entityManager.prepareResolutionManifest(callContext, securityContext, catalogName); if (key instanceof TableIdentifier identifier) { manifest.addPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), + new ResolverPath(PolarisCatalogHelpers.tableIdentifierToList(identifier), entityType), identifier); manifest.resolveAll(); - return manifest.getResolvedPath(identifier, subType); + return manifest.getResolvedPath(identifier, entityType, subType); } else { throw new IllegalStateException( String.format( - "Trying to getResolvedPath(key, subType) for %s with class %s and subType %s", - key, key.getClass(), subType)); + "Trying to getResolvedPath(key, subType) for %s with class %s and type %s / %s", + key, key.getClass(), entityType, subType)); } } @@ -122,17 +121,15 @@ public PolarisResolvedPathWrapper getPassthroughResolvedPath(Object key) { @Override public PolarisResolvedPathWrapper getPassthroughResolvedPath( - Object key, PolarisEntitySubType subType) { + Object key, PolarisEntityType entityType, PolarisEntitySubType subType) { PolarisResolutionManifest manifest = entityManager.prepareResolutionManifest(callContext, securityContext, catalogName); if (key instanceof TableIdentifier identifier) { manifest.addPassthroughPath( - new ResolverPath( - PolarisCatalogHelpers.tableIdentifierToList(identifier), - PolarisEntityType.TABLE_LIKE), + new ResolverPath(PolarisCatalogHelpers.tableIdentifierToList(identifier), entityType), identifier); - return manifest.getPassthroughResolvedPath(identifier, subType); + return manifest.getPassthroughResolvedPath(identifier, entityType, subType); } else { throw new IllegalStateException( String.format(