Skip to content

Commit 9d663b2

Browse files
eric-maynarderic-maynard
authored
Refactor GenericTableCatalog to support generic table federation (#1579)
* initial commit * stable * javadocs * autolint * changes per review
1 parent 499bae1 commit 9d663b2

File tree

6 files changed

+224
-162
lines changed

6 files changed

+224
-162
lines changed

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@
8080
import org.apache.polaris.core.secrets.UserSecretsManagerFactory;
8181
import org.apache.polaris.service.admin.PolarisAdminService;
8282
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
83-
import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
83+
import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
8484
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
8585
import org.apache.polaris.service.catalog.io.FileIOFactory;
8686
import org.apache.polaris.service.catalog.policy.PolicyCatalog;
@@ -194,7 +194,7 @@ public Map<String, String> getConfigOverrides() {
194194
@Inject protected PolarisEventListener polarisEventListener;
195195

196196
protected IcebergCatalog baseCatalog;
197-
protected GenericTableCatalog genericTableCatalog;
197+
protected PolarisGenericTableCatalog genericTableCatalog;
198198
protected PolicyCatalog policyCatalog;
199199
protected PolarisAdminService adminService;
200200
protected PolarisEntityManager entityManager;
@@ -481,7 +481,8 @@ private void initBaseCatalog() {
481481
ImmutableMap.of(
482482
CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO"));
483483
this.genericTableCatalog =
484-
new GenericTableCatalog(metaStoreManager, callContext, passthroughView);
484+
new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView);
485+
this.genericTableCatalog.initialize(CATALOG_NAME, ImmutableMap.of());
485486
this.policyCatalog = new PolicyCatalog(metaStoreManager, callContext, passthroughView);
486487
}
487488

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogHandlerAuthzTest.java‎ renamed to ‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogHandlerAuthzTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import org.junit.jupiter.api.Test;
3232

3333
@QuarkusTest
34-
public class GenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase {
34+
public class PolarisGenericTableCatalogHandlerAuthzTest extends PolarisAuthzTestBase {
3535

3636
private GenericTableCatalogHandler newWrapper() {
3737
return newWrapper(Set.of());

‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/GenericTableCatalogTest.java‎ renamed to ‎quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@
7676
import org.apache.polaris.core.storage.cache.StorageCredentialCache;
7777
import org.apache.polaris.service.admin.PolarisAdminService;
7878
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
79-
import org.apache.polaris.service.catalog.generic.GenericTableCatalog;
79+
import org.apache.polaris.service.catalog.generic.PolarisGenericTableCatalog;
8080
import org.apache.polaris.service.catalog.iceberg.IcebergCatalog;
8181
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
8282
import org.apache.polaris.service.catalog.io.FileIOFactory;
@@ -98,8 +98,8 @@
9898
import software.amazon.awssdk.services.sts.model.Credentials;
9999

100100
@QuarkusTest
101-
@TestProfile(GenericTableCatalogTest.Profile.class)
102-
public class GenericTableCatalogTest {
101+
@TestProfile(PolarisGenericTableCatalogTest.Profile.class)
102+
public class PolarisGenericTableCatalogTest {
103103

104104
public static class Profile implements QuarkusTestProfile {
105105

@@ -128,7 +128,7 @@ public Map<String, String> getConfigOverrides() {
128128
@Inject PolarisStorageIntegrationProvider storageIntegrationProvider;
129129
@Inject PolarisDiagnostics diagServices;
130130

131-
private GenericTableCatalog genericTableCatalog;
131+
private PolarisGenericTableCatalog genericTableCatalog;
132132
private IcebergCatalog icebergCatalog;
133133
private CallContext callContext;
134134
private AwsStorageConfigInfo storageConfigModel;
@@ -262,7 +262,8 @@ public void before(TestInfo testInfo) {
262262
.thenReturn((PolarisStorageIntegration) storageIntegration);
263263

264264
this.genericTableCatalog =
265-
new GenericTableCatalog(metaStoreManager, callContext, passthroughView);
265+
new PolarisGenericTableCatalog(metaStoreManager, callContext, passthroughView);
266+
this.genericTableCatalog.initialize(CATALOG_NAME, Map.of());
266267
this.icebergCatalog =
267268
new IcebergCatalog(
268269
entityManager,

‎service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalog.java‎

Lines changed: 13 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -22,163 +22,24 @@
2222
import java.util.Map;
2323
import org.apache.iceberg.catalog.Namespace;
2424
import org.apache.iceberg.catalog.TableIdentifier;
25-
import org.apache.iceberg.exceptions.AlreadyExistsException;
26-
import org.apache.iceberg.exceptions.NoSuchNamespaceException;
27-
import org.apache.iceberg.exceptions.NoSuchTableException;
28-
import org.apache.polaris.core.catalog.PolarisCatalogHelpers;
29-
import org.apache.polaris.core.context.CallContext;
30-
import org.apache.polaris.core.entity.CatalogEntity;
31-
import org.apache.polaris.core.entity.PolarisEntity;
32-
import org.apache.polaris.core.entity.PolarisEntitySubType;
33-
import org.apache.polaris.core.entity.PolarisEntityType;
3425
import org.apache.polaris.core.entity.table.GenericTableEntity;
35-
import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
36-
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
37-
import org.apache.polaris.core.persistence.dao.entity.BaseResult;
38-
import org.apache.polaris.core.persistence.dao.entity.DropEntityResult;
39-
import org.apache.polaris.core.persistence.dao.entity.EntityResult;
40-
import org.apache.polaris.core.persistence.pagination.PageToken;
41-
import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
42-
import org.slf4j.Logger;
43-
import org.slf4j.LoggerFactory;
4426

45-
public class GenericTableCatalog {
46-
private static final Logger LOGGER = LoggerFactory.getLogger(GenericTableCatalog.class);
27+
/** A catalog for managing `GenericTableEntity` instances */
28+
public interface GenericTableCatalog {
4729

48-
private final CallContext callContext;
49-
private final PolarisResolutionManifestCatalogView resolvedEntityView;
50-
private final CatalogEntity catalogEntity;
51-
private long catalogId = -1;
52-
private PolarisMetaStoreManager metaStoreManager;
30+
/** Should be called before other methods */
31+
void initialize(String name, Map<String, String> properties);
5332

54-
public GenericTableCatalog(
55-
PolarisMetaStoreManager metaStoreManager,
56-
CallContext callContext,
57-
PolarisResolutionManifestCatalogView resolvedEntityView) {
58-
this.callContext = callContext;
59-
this.resolvedEntityView = resolvedEntityView;
60-
this.catalogEntity =
61-
CatalogEntity.of(resolvedEntityView.getResolvedReferenceCatalogEntity().getRawLeafEntity());
62-
this.catalogId = catalogEntity.getId();
63-
this.metaStoreManager = metaStoreManager;
64-
}
33+
/** Create a generic table with the specified identifier */
34+
GenericTableEntity createGenericTable(
35+
TableIdentifier tableIdentifier, String format, String doc, Map<String, String> properties);
6536

66-
public GenericTableEntity createGenericTable(
67-
TableIdentifier tableIdentifier, String format, String doc, Map<String, String> properties) {
68-
PolarisResolvedPathWrapper resolvedParent =
69-
resolvedEntityView.getResolvedPath(tableIdentifier.namespace());
70-
if (resolvedParent == null) {
71-
// Illegal state because the namespace should've already been in the static resolution set.
72-
throw new IllegalStateException(
73-
String.format(
74-
"Failed to fetch resolved parent for TableIdentifier '%s'", tableIdentifier));
75-
}
37+
/** Retrieve a generic table entity with a given identifier */
38+
GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier);
7639

77-
List<PolarisEntity> catalogPath = resolvedParent.getRawFullPath();
40+
/** Drop a generic table entity with a given identifier */
41+
boolean dropGenericTable(TableIdentifier tableIdentifier);
7842

79-
PolarisResolvedPathWrapper resolvedEntities =
80-
resolvedEntityView.getPassthroughResolvedPath(
81-
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ANY_SUBTYPE);
82-
GenericTableEntity entity =
83-
GenericTableEntity.of(
84-
resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity());
85-
if (null == entity) {
86-
entity =
87-
new GenericTableEntity.Builder(tableIdentifier, format)
88-
.setCatalogId(this.catalogId)
89-
.setParentNamespace(tableIdentifier.namespace())
90-
.setParentId(resolvedParent.getRawLeafEntity().getId())
91-
.setId(
92-
this.metaStoreManager
93-
.generateNewEntityId(this.callContext.getPolarisCallContext())
94-
.getId())
95-
.setProperties(properties)
96-
.setDoc(doc)
97-
.setCreateTimestamp(System.currentTimeMillis())
98-
.build();
99-
} else {
100-
throw new AlreadyExistsException(
101-
"Iceberg table, view, or generic table already exists: %s", tableIdentifier);
102-
}
103-
104-
EntityResult res =
105-
this.metaStoreManager.createEntityIfNotExists(
106-
this.callContext.getPolarisCallContext(),
107-
PolarisEntity.toCoreList(catalogPath),
108-
entity);
109-
if (!res.isSuccess()) {
110-
switch (res.getReturnStatus()) {
111-
case BaseResult.ReturnStatus.ENTITY_ALREADY_EXISTS:
112-
throw new AlreadyExistsException(
113-
"Iceberg table, view, or generic table already exists: %s", tableIdentifier);
114-
115-
default:
116-
throw new IllegalStateException(
117-
String.format(
118-
"Unknown error status for identifier %s: %s with extraInfo: %s",
119-
tableIdentifier, res.getReturnStatus(), res.getExtraInformation()));
120-
}
121-
}
122-
GenericTableEntity resultEntity = GenericTableEntity.of(res.getEntity());
123-
LOGGER.debug(
124-
"Created GenericTable entity {} with TableIdentifier {}", resultEntity, tableIdentifier);
125-
return resultEntity;
126-
}
127-
128-
public GenericTableEntity loadGenericTable(TableIdentifier tableIdentifier) {
129-
PolarisResolvedPathWrapper resolvedEntities =
130-
resolvedEntityView.getPassthroughResolvedPath(
131-
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE);
132-
GenericTableEntity entity =
133-
GenericTableEntity.of(
134-
resolvedEntities == null ? null : resolvedEntities.getRawLeafEntity());
135-
if (null == entity) {
136-
throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier);
137-
} else {
138-
return entity;
139-
}
140-
}
141-
142-
public boolean dropGenericTable(TableIdentifier tableIdentifier) {
143-
PolarisResolvedPathWrapper resolvedEntities =
144-
resolvedEntityView.getPassthroughResolvedPath(
145-
tableIdentifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.GENERIC_TABLE);
146-
147-
if (resolvedEntities == null) {
148-
throw new NoSuchTableException("Generic table does not exist: %s", tableIdentifier);
149-
}
150-
151-
List<PolarisEntity> catalogPath = resolvedEntities.getRawParentPath();
152-
PolarisEntity leafEntity = resolvedEntities.getRawLeafEntity();
153-
154-
DropEntityResult dropEntityResult =
155-
this.metaStoreManager.dropEntityIfExists(
156-
this.callContext.getPolarisCallContext(),
157-
PolarisEntity.toCoreList(catalogPath),
158-
leafEntity,
159-
Map.of(),
160-
false);
161-
162-
return dropEntityResult.isSuccess();
163-
}
164-
165-
public List<TableIdentifier> listGenericTables(Namespace namespace) {
166-
PolarisResolvedPathWrapper resolvedEntities = resolvedEntityView.getResolvedPath(namespace);
167-
if (resolvedEntities == null) {
168-
throw new NoSuchNamespaceException("Namespace '%s' does not exist", namespace);
169-
}
170-
171-
List<PolarisEntity> catalogPath = resolvedEntities.getRawFullPath();
172-
List<PolarisEntity.NameAndId> entities =
173-
PolarisEntity.toNameAndIdList(
174-
this.metaStoreManager
175-
.listEntities(
176-
this.callContext.getPolarisCallContext(),
177-
PolarisEntity.toCoreList(catalogPath),
178-
PolarisEntityType.TABLE_LIKE,
179-
PolarisEntitySubType.GENERIC_TABLE,
180-
PageToken.readEverything())
181-
.getEntities());
182-
return PolarisCatalogHelpers.nameAndIdToTableIdentifiers(catalogPath, entities);
183-
}
43+
/** List all generic tables under a specific namespace */
44+
List<TableIdentifier> listGenericTables(Namespace namespace);
18445
}

‎service/common/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogHandler.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ public GenericTableCatalogHandler(
5555
@Override
5656
protected void initializeCatalog() {
5757
this.genericTableCatalog =
58-
new GenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest);
58+
new PolarisGenericTableCatalog(metaStoreManager, callContext, this.resolutionManifest);
59+
this.genericTableCatalog.initialize(catalogName, Map.of());
5960
}
6061

6162
public ListGenericTablesResponse listGenericTables(Namespace parent) {

0 commit comments

Comments
 (0)