diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index b915790111..a695922d74 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -114,7 +114,6 @@ polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE"] polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # realm overrides -# polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true # polaris.persistence.type=eclipse-link diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java index 3bd8f45aaf..cbff07e4c7 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogTest.java @@ -99,6 +99,7 @@ import org.apache.polaris.core.persistence.dao.entity.EntityResult; import org.apache.polaris.core.persistence.dao.entity.PrincipalSecretsResult; import org.apache.polaris.core.persistence.pagination.PageToken; +import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView; import org.apache.polaris.core.persistence.transactional.TransactionalPersistence; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; @@ -173,8 +174,6 @@ public Map getConfigOverrides() { "true", "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true", - "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", - "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]", "polaris.features.\"LIST_PAGINATION_ENABLED\"", @@ -227,6 +226,45 @@ public Map getConfigOverrides() { private TestPolarisEventListener testPolarisEventListener; private ReservedProperties reservedProperties; + /** + * A subclass of IcebergCatalog that adds FileIO management capabilities. This allows the file IO + * logic to be encapsulated in a dedicated class. + */ + public static class IcebergFileIOCatalog extends IcebergCatalog { + + public IcebergFileIOCatalog( + PolarisEntityManager entityManager, + PolarisMetaStoreManager metaStoreManager, + CallContext callContext, + PolarisResolutionManifestCatalogView resolvedEntityView, + SecurityContext securityContext, + TaskExecutor taskExecutor, + FileIOFactory fileIOFactory, + PolarisEventListener polarisEventListener) { + super( + entityManager, + metaStoreManager, + callContext, + resolvedEntityView, + securityContext, + taskExecutor, + fileIOFactory, + polarisEventListener); + } + + @Override + public synchronized FileIO getIo() { + if (catalogFileIO == null) { + catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); + if (closeableGroup != null) { + closeableGroup.addCloseable(catalogFileIO); + } + } + + return catalogFileIO; + } + } + @BeforeAll public static void setUpMocks() { PolarisStorageIntegrationProviderImpl mock = @@ -368,7 +406,7 @@ protected IcebergCatalog initCatalog( callContext, entityManager, securityContext, CATALOG_NAME); TaskExecutor taskExecutor = Mockito.mock(); IcebergCatalog icebergCatalog = - new IcebergCatalog( + new IcebergFileIOCatalog( entityManager, metaStoreManager, callContext, @@ -1000,7 +1038,7 @@ public void testUpdateNotificationCreateTableWithLocalFilePrefix() { callContext, entityManager, securityContext, catalogWithoutStorage); TaskExecutor taskExecutor = Mockito.mock(); IcebergCatalog catalog = - new IcebergCatalog( + new IcebergFileIOCatalog( entityManager, metaStoreManager, callContext, @@ -1067,7 +1105,7 @@ public void testUpdateNotificationCreateTableWithHttpPrefix() { callContext, entityManager, securityContext, catalogName); TaskExecutor taskExecutor = Mockito.mock(); IcebergCatalog catalog = - new IcebergCatalog( + new IcebergFileIOCatalog( entityManager, metaStoreManager, callContext, diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java index 0ac72a1b51..8d7924b306 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogViewTest.java @@ -108,8 +108,6 @@ public Map getConfigOverrides() { "true", "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true", - "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", - "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]", "polaris.event-listener.type", diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java index 267fbb3867..f2e04df415 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisGenericTableCatalogTest.java @@ -110,8 +110,6 @@ public Map getConfigOverrides() { "true", "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true", - "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", - "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]", "polaris.readiness.ignore-severe-issues", diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java index e94f70c636..be1ce47721 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolicyCatalogTest.java @@ -123,8 +123,6 @@ public Map getConfigOverrides() { return Map.of( "polaris.features.\"ALLOW_SPECIFYING_FILE_IO_IMPL\"", "true", - "polaris.features.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", - "true", "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]", "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index ba9df38c96..e39a4d0acb 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -91,7 +91,6 @@ import org.apache.polaris.core.catalog.PolarisCatalogHelpers; import org.apache.polaris.core.config.BehaviorChangeConfiguration; import org.apache.polaris.core.config.FeatureConfiguration; -import org.apache.polaris.core.config.PolarisConfiguration; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.NamespaceEntity; @@ -170,14 +169,15 @@ public class IcebergCatalog extends BaseMetastoreViewCatalog private final SecurityContext securityContext; private final PolarisEventListener polarisEventListener; - private String ioImplClassName; - private FileIO catalogFileIO; + protected String ioImplClassName; + protected FileIO catalogFileIO; + protected CloseableGroup closeableGroup; + protected Map tableDefaultProperties; + private final String catalogName; private long catalogId = -1; private String defaultBaseLocation; - private CloseableGroup closeableGroup; private Map catalogProperties; - private Map tableDefaultProperties; private FileIOFactory fileIOFactory; private PolarisMetaStoreManager metaStoreManager; @@ -263,16 +263,6 @@ public void initialize(String name, Map properties) { tableDefaultProperties = PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); - if (initializeDefaultCatalogFileioForTest()) { - LOGGER.debug( - "Initializing a default catalogFileIO with properties {}", tableDefaultProperties); - this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); - closeableGroup.addCloseable(this.catalogFileIO); - } else { - LOGGER.debug("Not initializing default catalogFileIO"); - this.catalogFileIO = null; - } - callContext.closeables().addCloseable(this); this.closeableGroup = new CloseableGroup(); closeableGroup.addCloseable(metricsReporter()); @@ -280,16 +270,6 @@ public void initialize(String name, Map properties) { tableDefaultProperties = PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); - - if (initializeDefaultCatalogFileioForTest()) { - LOGGER.debug( - "Initializing a default catalogFileIO with properties {}", tableDefaultProperties); - this.catalogFileIO = loadFileIO(ioImplClassName, tableDefaultProperties); - closeableGroup.addCloseable(this.catalogFileIO); - } else { - LOGGER.debug("Not initializing default catalogFileIO"); - this.catalogFileIO = null; - } } public void setMetaStoreManager(PolarisMetaStoreManager newMetaStoreManager) { @@ -359,8 +339,7 @@ public ViewBuilder buildView(TableIdentifier identifier) { @VisibleForTesting public TableOperations newTableOps( TableIdentifier tableIdentifier, boolean makeMetadataCurrentOnCommit) { - return new BasePolarisTableOperations( - catalogFileIO, tableIdentifier, makeMetadataCurrentOnCommit); + return new BasePolarisTableOperations(getIo(), tableIdentifier, makeMetadataCurrentOnCommit); } @Override @@ -862,9 +841,10 @@ private Page listViews(Namespace namespace, PageToken pageToken return listTableLike(PolarisEntitySubType.ICEBERG_VIEW, namespace, pageToken); } + @VisibleForTesting @Override protected ViewOperations newViewOps(TableIdentifier identifier) { - return new BasePolarisViewOperations(catalogFileIO, identifier); + return new BasePolarisViewOperations(getIo(), identifier); } @Override @@ -2514,7 +2494,7 @@ private Page listTableLike( * @param properties used to initialize the FileIO implementation * @return FileIO object */ - private FileIO loadFileIO(String ioImpl, Map properties) { + protected FileIO loadFileIO(String ioImpl, Map properties) { IcebergTableLikeEntity icebergTableLikeEntity = IcebergTableLikeEntity.of(catalogEntity); TableIdentifier identifier = icebergTableLikeEntity.getTableIdentifier(); Set locations = Set.of(catalogEntity.getDefaultBaseLocation()); @@ -2545,12 +2525,6 @@ private Boolean getBooleanContextConfiguration(String configKey, boolean default .getConfiguration(callContext.getPolarisCallContext(), configKey, defaultValue); } - private boolean initializeDefaultCatalogFileioForTest() { - var ctx = callContext.getPolarisCallContext(); - return ctx.getConfigurationStore() - .getConfiguration(ctx, INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST); - } - private int getMaxMetadataRefreshRetries() { var ctx = callContext.getPolarisCallContext(); return ctx.getConfigurationStore() @@ -2574,11 +2548,4 @@ private PageToken buildPageToken(@Nullable String tokenString, @Nullable Integer return PageToken.build(tokenString, pageSize); } } - - static final FeatureConfiguration INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST = - PolarisConfiguration.builder() - .key("INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST") - .defaultValue(false) - .description("") - .buildFeatureConfiguration(); } diff --git a/site/content/in-dev/unreleased/configuration.md b/site/content/in-dev/unreleased/configuration.md index e9802c2b28..b195d05a5b 100644 --- a/site/content/in-dev/unreleased/configuration.md +++ b/site/content/in-dev/unreleased/configuration.md @@ -90,7 +90,6 @@ read-only mode, as Polaris only reads the configuration file once, at startup. | `polaris.realm-context.header-name` | `Polaris-Realm` | Define the header name defining the realm context. | | `polaris.features."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"` | `false` | Flag to enforce check if credential rotation. | | `polaris.features."SUPPORTED_CATALOG_STORAGE_TYPES"` | `FILE` | Define the catalog supported storage. Supported values are `S3`, `GCS`, `AZURE`, `FILE`. | -| `polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"` | `true` | "Override" realm features, here the catalog init default flag. | | `polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"` | `true` | "Override" realm features, here the skip credential subscoping indirection flag. | | `polaris.authentication.authenticator.type` | `default` | Define the Polaris authenticator type. | | `polaris.authentication.token-service.type` | `default` | Define the Polaris token service type. |