Skip to content

Refactor GenericTableCatalog to support generic table federation #1579

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 14, 2025
Merged

Refactor GenericTableCatalog to support generic table federation #1579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
69337a0
initial commit
eric-maynard May 13, 2025
e5c10d0
stable
eric-maynard May 13, 2025
6c68f18
javadocs
eric-maynard May 13, 2025
4feb77b
autolint
eric-maynard May 13, 2025
d2ecece
Merge branch 'main' of github.com:apache/polaris into generic-federation
eric-maynard May 14, 2025
ebb75f7
changes per review
eric-maynard May 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .../service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
import org.apache.polaris.core.secrets.UserSecretsManagerFactory;
import org.apache.polaris.service.admin.PolarisAdminService;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
import org.apache.polaris.service.catalog.io.FileIOFactory;
import org.apache.polaris.service.catalog.policy.PolicyCatalog;
Expand Down Expand Up @@ -194,7 +194,7 @@ public Map<String, String> getConfigOverrides() {
@Inject protected PolarisEventListener polarisEventListener;

protected IcebergCatalog baseCatalog;
protected GenericTableCatalog genericTableCatalog;
protected PolarisGenericTableCatalog genericTableCatalog;
protected PolicyCatalog policyCatalog;
protected PolarisAdminService adminService;
protected PolarisEntityManager entityManager;
Expand Down Expand Up @@ -481,7 +481,8 @@ private void initBaseCatalog() {
ImmutableMap.of(
CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO"));
this.genericTableCatalog =
new GenericTableCatalog(metaStoreManager, callContext, passthroughView);
new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView);
this.genericTableCatalog.initialize(CATALOG_NAME, ImmutableMap.of());
this.policyCatalog = new PolicyCatalog(metaStoreManager, callContext, passthroughView);
}

Expand Down
2 changes: 1 addition & 1 deletion .../GenericTableCatalogHandlerAuthzTest.java → ...sGenericTableCatalogHandlerAuthzTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.junit.jupiter.api.Test;

@QuarkusTest
public class GenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase {
public class PolarisGenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase {

private GenericTableCatalogHandler newWrapper() {
return newWrapper(Set.of());
Expand Down
11 changes: 6 additions & 5 deletions ...rkus/catalog/GenericTableCatalogTest.java → ...talog/PolarisGenericTableCatalogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
import org.apache.polaris.service.admin.PolarisAdminService;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
import org.apache.polaris.service.catalog.io.FileIOFactory;
Expand All @@ -98,8 +98,8 @@
import software.amazon.awssdk.services.sts.model.Credentials;

@QuarkusTest
@TestProfile(GenericTableCatalogTest.Profile.class)
public class GenericTableCatalogTest {
@TestProfile(PolarisGenericTableCatalogTest.Profile.class)
public class PolarisGenericTableCatalogTest {

public static class Profile implements QuarkusTestProfile {

Expand Down Expand Up @@ -128,7 +128,7 @@ public Map<String, String> getConfigOverrides() {
@Inject PolarisStorageIntegrationProvider storageIntegrationProvider;
@Inject PolarisDiagnostics diagServices;

private GenericTableCatalog genericTableCatalog;
private PolarisGenericTableCatalog genericTableCatalog;
private IcebergCatalog icebergCatalog;
private CallContext callContext;
private AwsStorageConfigInfo storageConfigModel;
Expand Down Expand Up @@ -262,7 +262,8 @@ public void before(TestInfo testInfo) {
.thenReturn((PolarisStorageIntegration) storageIntegration);

this.genericTableCatalog =
new GenericTableCatalog(metaStoreManager, callContext, passthroughView);
new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView);
this.genericTableCatalog.initialize(CATALOG_NAME, Map.of());
this.icebergCatalog =
new IcebergCatalog(
entityManager,
Expand Down
165 changes: 13 additions & 152 deletions .../common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java
View file
Open in desktop
Copy link
Contributor Author

@eric-maynard eric-maynard May 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This used to be a concrete implementation, but is becoming an interface

Copy link
Contributor

@singhpk234 singhpk234 May 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the idea !
wondering if you are open to addingname() and initialize(String name, Map<String, String> properties) like the Catalog interface of iceberg.

Copy link
Contributor Author

@eric-maynard eric-maynard May 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think it's a good idea -- for federation we'll definitely want some kind of initialize

Original file line number Diff line number Diff line change
Expand Up @@ -22,163 +22,24 @@
import java.util.Map;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
import org.apache.iceberg.exceptions.NoSuchTableException;
import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
import org.apache.polaris.core.context.CallContext;
import org.apache.polaris.core.entity.CatalogEntity;
import org.apache.polaris.core.entity.PolarisEntity;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.table.GenericTableEntity;
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.dao.entity.BaseResult;
import org.apache.polaris.core.persistence.dao.entity.DropEntityResult;
import org.apache.polaris.core.persistence.dao.entity.EntityResult;
import org.apache.polaris.core.persistence.pagination.PageToken;
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class GenericTableCatalog {
private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalog.class);
/** A catalog for managing `GenericTableEntity` instances */
public interface GenericTableCatalog {

private final CallContext callContext;
private final PolarisResolutionManifestCatalogView resolvedEntityView;
private final CatalogEntity catalogEntity;
private long catalogId = -1;
private PolarisMetaStoreManager metaStoreManager;
/** Should be called before other methods */
void initialize(String name, Map<String, String> properties);

public GenericTableCatalog(
PolarisMetaStoreManager metaStoreManager,
CallContext callContext,
PolarisResolutionManifestCatalogView resolvedEntityView) {
this.callContext = callContext;
this.resolvedEntityView = resolvedEntityView;
this.catalogEntity =
CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity());
this.catalogId = catalogEntity.getId();
this.metaStoreManager = metaStoreManager;
}
/** Create a generic table with the specified identifier */
GenericTableEntity createGenericTable(
TableIdentifier tableIdentifier, String format, String doc, Map<String, String> properties);

public GenericTableEntity createGenericTable(
TableIdentifier tableIdentifier, String format, String doc, Map<String, String> properties) {
PolarisResolvedPathWrapper resolvedParent =
resolvedEntityView.getResolvedPath(tableIdentifier.namespace());
if (resolvedParent == null) {
// Illegal state because the namespace should've already been in the static resolution set.
throw new IllegalStateException(
String.format(
"Failed to fetch resolved parent for TableIdentifier '%s'", tableIdentifier));
}
/** Retrieve a generic table entity with a given identifier */
GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier);

List<PolarisEntity> catalogPath = resolvedParent.getRawFullPath();
/** Drop a generic table entity with a given identifier */
boolean dropGenericTable(TableIdentifier tableIdentifier);

PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getPassthroughResolvedPath(
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE);
GenericTableEntity entity =
GenericTableEntity.of(
resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity());
if (null == entity) {
entity =
new GenericTableEntity.Builder(tableIdentifier, format)
.setCatalogId(this.catalogId)
.setParentNamespace(tableIdentifier.namespace())
.setParentId(resolvedParent.getRawLeafEntity().getId())
.setId(
this.metaStoreManager
.generateNewEntityId(this.callContext.getPolarisCallContext())
.getId())
.setProperties(properties)
.setDoc(doc)
.setCreateTimestamp(System.currentTimeMillis())
.build();
} else {
throw new AlreadyExistsException(
"Iceberg table, view, or generic table already exists: %s", tableIdentifier);
}

EntityResult res =
this.metaStoreManager.createEntityIfNotExists(
this.callContext.getPolarisCallContext(),
PolarisEntity.toCoreList(catalogPath),
entity);
if (!res.isSuccess()) {
switch (res.getReturnStatus()) {
case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS:
throw new AlreadyExistsException(
"Iceberg table, view, or generic table already exists: %s", tableIdentifier);

default:
throw new IllegalStateException(
String.format(
"Unknown error status for identifier %s: %s with extraInfo: %s",
tableIdentifier, res.getReturnStatus(), res.getExtraInformation()));
}
}
GenericTableEntity resultEntity = GenericTableEntity.of(res.getEntity());
LOGGER.debug(
"Created GenericTable entity {} with TableIdentifier {}", resultEntity, tableIdentifier);
return resultEntity;
}

public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) {
PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getPassthroughResolvedPath(
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE);
GenericTableEntity entity =
GenericTableEntity.of(
resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity());
if (null == entity) {
throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier);
} else {
return entity;
}
}

public boolean dropGenericTable(TableIdentifier tableIdentifier) {
PolarisResolvedPathWrapper resolvedEntities =
resolvedEntityView.getPassthroughResolvedPath(
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE);

if (resolvedEntities == null) {
throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier);
}

List<PolarisEntity> catalogPath = resolvedEntities.getRawParentPath();
PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity();

DropEntityResult dropEntityResult =
this.metaStoreManager.dropEntityIfExists(
this.callContext.getPolarisCallContext(),
PolarisEntity.toCoreList(catalogPath),
leafEntity,
Map.of(),
false);

return dropEntityResult.isSuccess();
}

public List<TableIdentifier> listGenericTables(Namespace namespace) {
PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace);
if (resolvedEntities == null) {
throw new NoSuchNamespaceException("Namespace '%s' does not exist", namespace);
}

List<PolarisEntity> catalogPath = resolvedEntities.getRawFullPath();
List<PolarisEntity.NameAndId> entities =
PolarisEntity.toNameAndIdList(
this.metaStoreManager
.listEntities(
this.callContext.getPolarisCallContext(),
PolarisEntity.toCoreList(catalogPath),
PolarisEntityType.TABLE_LIKE,
PolarisEntitySubType.GENERIC_TABLE,
PageToken.readEverything())
.getEntities());
return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities);
}
/** List all generic tables under a specific namespace */
List<TableIdentifier> listGenericTables(Namespace namespace);
}
3 changes: 2 additions & 1 deletion .../src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ public GenericTableCatalogHandler(
@Override
protected void initializeCatalog() {
this.genericTableCatalog =
new GenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest);
new PolarisGenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest);
this.genericTableCatalog.initialize(catalogName, Map.of());
}

public ListGenericTablesResponse listGenericTables(Namespace parent) {
Expand Down
Loading