diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java index b5ef68a6bd..37dcc04e54 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java @@ -80,7 +80,7 @@ import org.apache.polaris.core.secrets.UserSecretsManagerFactory; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.generic.GenericTableCatalog; +import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.policy.PolicyCatalog; @@ -194,7 +194,7 @@ public Map getConfigOverrides() { @Inject protected PolarisEventListener polarisEventListener; protected IcebergCatalog baseCatalog; - protected GenericTableCatalog genericTableCatalog; + protected PolarisGenericTableCatalog genericTableCatalog; protected PolicyCatalog policyCatalog; protected PolarisAdminService adminService; protected PolarisEntityManager entityManager; @@ -481,7 +481,8 @@ private void initBaseCatalog() { ImmutableMap.of( CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); this.genericTableCatalog = - new GenericTableCatalog(metaStoreManager, callContext, passthroughView); + new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView); + this.genericTableCatalog.initialize(CATALOG_NAME, ImmutableMap.of()); this.policyCatalog = new PolicyCatalog(metaStoreManager, callContext, passthroughView); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java similarity index 99% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java index fb3114e69a..0f9fd31903 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java @@ -31,7 +31,7 @@ import org.junit.jupiter.api.Test; @QuarkusTest -public class GenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase { +public class PolarisGenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase { private GenericTableCatalogHandler newWrapper() { return newWrapper(Set.of()); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java similarity index 98% rename from quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java rename to quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 04c84c5f60..5f8c2d0284 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -76,7 +76,7 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; -import org.apache.polaris.service.catalog.generic.GenericTableCatalog; +import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.catalog.io.FileIOFactory; @@ -98,8 +98,8 @@ import software.amazon.awssdk.services.sts.model.Credentials; @QuarkusTest -@TestProfile(GenericTableCatalogTest.Profile.class) -public class GenericTableCatalogTest { +@TestProfile(PolarisGenericTableCatalogTest.Profile.class) +public class PolarisGenericTableCatalogTest { public static class Profile implements QuarkusTestProfile { @@ -128,7 +128,7 @@ public Map getConfigOverrides() { @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; @Inject PolarisDiagnostics diagServices; - private GenericTableCatalog genericTableCatalog; + private PolarisGenericTableCatalog genericTableCatalog; private IcebergCatalog icebergCatalog; private CallContext callContext; private AwsStorageConfigInfo storageConfigModel; @@ -262,7 +262,8 @@ public void before(TestInfo testInfo) { .thenReturn((PolarisStorageIntegration) storageIntegration); this.genericTableCatalog = - new GenericTableCatalog(metaStoreManager, callContext, passthroughView); + new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView); + this.genericTableCatalog.initialize(CATALOG_NAME, Map.of()); this.icebergCatalog = new IcebergCatalog( entityManager, diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java index 79adeaee85..197be36452 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java @@ -22,163 +22,24 @@ import java.util.Map; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.polaris.core.catalog.PolarisCatalogHelpers; -import org.apache.polaris.core.context.CallContext; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.core.entity.PolarisEntity; -import org.apache.polaris.core.entity.PolarisEntitySubType; -import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.table.GenericTableEntity; -import org.apache.polaris.core.persistence.PolarisMetaStoreManager; -import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; -import org.apache.polaris.core.persistence.dao.entity.BaseResult; -import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; -import org.apache.polaris.core.persistence.dao.entity.EntityResult; -import org.apache.polaris.core.persistence.pagination.PageToken; -import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class GenericTableCatalog { - private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalog.class); +/** A catalog for managing `GenericTableEntity` instances */ +public interface GenericTableCatalog { - private final CallContext callContext; - private final PolarisResolutionManifestCatalogView resolvedEntityView; - private final CatalogEntity catalogEntity; - private long catalogId = -1; - private PolarisMetaStoreManager metaStoreManager; + /** Should be called before other methods */ + void initialize(String name, Map properties); - public GenericTableCatalog( - PolarisMetaStoreManager metaStoreManager, - CallContext callContext, - PolarisResolutionManifestCatalogView resolvedEntityView) { - this.callContext = callContext; - this.resolvedEntityView = resolvedEntityView; - this.catalogEntity = - CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity()); - this.catalogId = catalogEntity.getId(); - this.metaStoreManager = metaStoreManager; - } + /** Create a generic table with the specified identifier */ + GenericTableEntity createGenericTable( + TableIdentifier tableIdentifier, String format, String doc, Map properties); - public GenericTableEntity createGenericTable( - TableIdentifier tableIdentifier, String format, String doc, Map properties) { - PolarisResolvedPathWrapper resolvedParent = - resolvedEntityView.getResolvedPath(tableIdentifier.namespace()); - if (resolvedParent == null) { - // Illegal state because the namespace should've already been in the static resolution set. - throw new IllegalStateException( - String.format( - "Failed to fetch resolved parent for TableIdentifier '%s'", tableIdentifier)); - } + /** Retrieve a generic table entity with a given identifier */ + GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier); - List catalogPath = resolvedParent.getRawFullPath(); + /** Drop a generic table entity with a given identifier */ + boolean dropGenericTable(TableIdentifier tableIdentifier); - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); - GenericTableEntity entity = - GenericTableEntity.of( - resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); - if (null == entity) { - entity = - new GenericTableEntity.Builder(tableIdentifier, format) - .setCatalogId(this.catalogId) - .setParentNamespace(tableIdentifier.namespace()) - .setParentId(resolvedParent.getRawLeafEntity().getId()) - .setId( - this.metaStoreManager - .generateNewEntityId(this.callContext.getPolarisCallContext()) - .getId()) - .setProperties(properties) - .setDoc(doc) - .setCreateTimestamp(System.currentTimeMillis()) - .build(); - } else { - throw new AlreadyExistsException( - "Iceberg table, view, or generic table already exists: %s", tableIdentifier); - } - - EntityResult res = - this.metaStoreManager.createEntityIfNotExists( - this.callContext.getPolarisCallContext(), - PolarisEntity.toCoreList(catalogPath), - entity); - if (!res.isSuccess()) { - switch (res.getReturnStatus()) { - case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS: - throw new AlreadyExistsException( - "Iceberg table, view, or generic table already exists: %s", tableIdentifier); - - default: - throw new IllegalStateException( - String.format( - "Unknown error status for identifier %s: %s with extraInfo: %s", - tableIdentifier, res.getReturnStatus(), res.getExtraInformation())); - } - } - GenericTableEntity resultEntity = GenericTableEntity.of(res.getEntity()); - LOGGER.debug( - "Created GenericTable entity {} with TableIdentifier {}", resultEntity, tableIdentifier); - return resultEntity; - } - - public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE); - GenericTableEntity entity = - GenericTableEntity.of( - resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); - if (null == entity) { - throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier); - } else { - return entity; - } - } - - public boolean dropGenericTable(TableIdentifier tableIdentifier) { - PolarisResolvedPathWrapper resolvedEntities = - resolvedEntityView.getPassthroughResolvedPath( - tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE); - - if (resolvedEntities == null) { - throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier); - } - - List catalogPath = resolvedEntities.getRawParentPath(); - PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); - - DropEntityResult dropEntityResult = - this.metaStoreManager.dropEntityIfExists( - this.callContext.getPolarisCallContext(), - PolarisEntity.toCoreList(catalogPath), - leafEntity, - Map.of(), - false); - - return dropEntityResult.isSuccess(); - } - - public List listGenericTables(Namespace namespace) { - PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); - if (resolvedEntities == null) { - throw new NoSuchNamespaceException("Namespace '%s' does not exist", namespace); - } - - List catalogPath = resolvedEntities.getRawFullPath(); - List entities = - PolarisEntity.toNameAndIdList( - this.metaStoreManager - .listEntities( - this.callContext.getPolarisCallContext(), - PolarisEntity.toCoreList(catalogPath), - PolarisEntityType.TABLE_LIKE, - PolarisEntitySubType.GENERIC_TABLE, - PageToken.readEverything()) - .getEntities()); - return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); - } + /** List all generic tables under a specific namespace */ + List listGenericTables(Namespace namespace); } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java index 126023c2b2..66a2b81d19 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java @@ -55,7 +55,8 @@ public GenericTableCatalogHandler( @Override protected void initializeCatalog() { this.genericTableCatalog = - new GenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest); + new PolarisGenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest); + this.genericTableCatalog.initialize(catalogName, Map.of()); } public ListGenericTablesResponse listGenericTables(Namespace parent) { diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java new file mode 100644 index 0000000000..2b884e787f --- /dev/null +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/generic/PolarisGenericTableCatalog.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.catalog.generic; + +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.polaris.core.catalog.PolarisCatalogHelpers; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.table.GenericTableEntity; +import org.apache.polaris.core.persistence.PolarisMetaStoreManager; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.dao.entity.BaseResult; +import org.apache.polaris.core.persistence.dao.entity.DropEntityResult; +import org.apache.polaris.core.persistence.dao.entity.EntityResult; +import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PolarisGenericTableCatalog implements GenericTableCatalog { + private static final Logger LOGGER = LoggerFactory.getLogger(PolarisGenericTableCatalog.class); + + private String name; + + private final CallContext callContext; + private final PolarisResolutionManifestCatalogView resolvedEntityView; + private final CatalogEntity catalogEntity; + private long catalogId = -1; + private PolarisMetaStoreManager metaStoreManager; + + public PolarisGenericTableCatalog( + PolarisMetaStoreManager metaStoreManager, + CallContext callContext, + PolarisResolutionManifestCatalogView resolvedEntityView) { + this.callContext = callContext; + this.resolvedEntityView = resolvedEntityView; + this.catalogEntity = + CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + this.catalogId = catalogEntity.getId(); + this.metaStoreManager = metaStoreManager; + } + + @Override + public void initialize(String name, Map properties) { + this.name = name; + if (!properties.isEmpty()) { + throw new IllegalStateException("PolarisGenericTableCatalog does not support properties"); + } + } + + @Override + public GenericTableEntity createGenericTable( + TableIdentifier tableIdentifier, String format, String doc, Map properties) { + PolarisResolvedPathWrapper resolvedParent = + resolvedEntityView.getResolvedPath(tableIdentifier.namespace()); + if (resolvedParent == null) { + // Illegal state because the namespace should've already been in the static resolution set. + throw new IllegalStateException( + String.format( + "Failed to fetch resolved parent for TableIdentifier '%s'", tableIdentifier)); + } + + List catalogPath = resolvedParent.getRawFullPath(); + + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE); + GenericTableEntity entity = + GenericTableEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + if (null == entity) { + entity = + new GenericTableEntity.Builder(tableIdentifier, format) + .setCatalogId(this.catalogId) + .setParentNamespace(tableIdentifier.namespace()) + .setParentId(resolvedParent.getRawLeafEntity().getId()) + .setId( + this.metaStoreManager + .generateNewEntityId(this.callContext.getPolarisCallContext()) + .getId()) + .setProperties(properties) + .setDoc(doc) + .setCreateTimestamp(System.currentTimeMillis()) + .build(); + } else { + throw new AlreadyExistsException( + "Iceberg table, view, or generic table already exists: %s", tableIdentifier); + } + + EntityResult res = + this.metaStoreManager.createEntityIfNotExists( + this.callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + entity); + if (!res.isSuccess()) { + switch (res.getReturnStatus()) { + case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS: + throw new AlreadyExistsException( + "Iceberg table, view, or generic table already exists: %s", tableIdentifier); + + default: + throw new IllegalStateException( + String.format( + "Unknown error status for identifier %s: %s with extraInfo: %s", + tableIdentifier, res.getReturnStatus(), res.getExtraInformation())); + } + } + GenericTableEntity resultEntity = GenericTableEntity.of(res.getEntity()); + LOGGER.debug( + "Created GenericTable entity {} with TableIdentifier {}", resultEntity, tableIdentifier); + return resultEntity; + } + + @Override + public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE); + GenericTableEntity entity = + GenericTableEntity.of( + resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity()); + if (null == entity) { + throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier); + } else { + return entity; + } + } + + @Override + public boolean dropGenericTable(TableIdentifier tableIdentifier) { + PolarisResolvedPathWrapper resolvedEntities = + resolvedEntityView.getPassthroughResolvedPath( + tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE); + + if (resolvedEntities == null) { + throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier); + } + + List catalogPath = resolvedEntities.getRawParentPath(); + PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity(); + + DropEntityResult dropEntityResult = + this.metaStoreManager.dropEntityIfExists( + this.callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + leafEntity, + Map.of(), + false); + + return dropEntityResult.isSuccess(); + } + + @Override + public List listGenericTables(Namespace namespace) { + PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace); + if (resolvedEntities == null) { + throw new NoSuchNamespaceException("Namespace '%s' does not exist", namespace); + } + + List catalogPath = resolvedEntities.getRawFullPath(); + List entities = + PolarisEntity.toNameAndIdList( + this.metaStoreManager + .listEntities( + this.callContext.getPolarisCallContext(), + PolarisEntity.toCoreList(catalogPath), + PolarisEntityType.TABLE_LIKE, + PolarisEntitySubType.GENERIC_TABLE, + PageToken.readEverything()) + .getEntities()); + return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities); + } +}