From 025eaad1df643680be4eb8c248affd191dbaaf6e Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Wed, 19 Feb 2025 05:26:52 +0000 Subject: [PATCH 1/5] API Spec: Add ConnectionConfigInfo to ExternalCatalog Remove the currently unused remoteUrl field from the top-level ExternalCatalog into the ConnectionConfigInfo as remoteUri instead for better consistency; remote catalogs in the future may be defined by arbitrary URIs that are not, for example, http(s) URLs. This is just the spec definition for now, so it's not yet wired into the internal entity layer or persistence objects. Allow extensibility of different connection types in the future even if we start with only an ICEBERG_REST type. Similarly, provide extensibility for different authn mechanisms to use with the connection. --- .../PolarisApplicationIntegrationTest.java | 1 - ...larisManagementServiceIntegrationTest.java | 3 - .../it/test/PolarisSparkIntegrationTest.java | 1 - .../polaris/core/entity/CatalogEntity.java | 5 -- spec/polaris-management-service.yml | 89 ++++++++++++++++++- 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 7a2961e402..95483dce43 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -210,7 +210,6 @@ private static void createCatalog( .setStorageConfigInfo(storageConfig) .build() : ExternalCatalog.builder() - .setRemoteUrl("http://faraway.com") .setName(catalogName) .setType(catalogType) .setProperties(props) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index e830a18ab4..82f88bd803 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -465,12 +465,10 @@ public void testCreateExternalCatalog() { .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) .build(); String catalogName = client.newEntityName("my-external-catalog"); - String remoteUrl = "http://localhost:8080"; Catalog catalog = ExternalCatalog.builder() .setType(Catalog.TypeEnum.EXTERNAL) .setName(catalogName) - .setRemoteUrl(remoteUrl) .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); @@ -483,7 +481,6 @@ public void testCreateExternalCatalog() { .isNotNull() .isInstanceOf(ExternalCatalog.class) .asInstanceOf(InstanceOfAssertFactories.type(ExternalCatalog.class)) - .returns(remoteUrl, ExternalCatalog::getRemoteUrl) .extracting(ExternalCatalog::getStorageConfigInfo) .isNotNull() .isInstanceOf(AwsStorageConfigInfo.class) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java index e7fa2dcece..db77df2750 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java @@ -167,7 +167,6 @@ public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentia .setName(externalCatalogName) .setProperties(externalProps) .setStorageConfigInfo(awsConfigModel) - .setRemoteUrl("http://dummy_url") .build(); managementApi.createCatalog(externalCatalog); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index f3bfd6edf0..00aab864e7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -63,7 +63,6 @@ public class CatalogEntity extends PolarisEntity { // translated into "s3://my-bucket/base/location/ns1/ns2/table1". public static final String REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY = "replace-new-location-prefix-with-catalog-default"; - public static final String REMOTE_URL = "remoteUrl"; public CatalogEntity(PolarisBaseEntity sourceEntity) { super(sourceEntity); @@ -84,9 +83,6 @@ public static CatalogEntity fromCatalog(Catalog catalog) { .setProperties(catalog.getProperties().toMap()) .setCatalogType(catalog.getType().name()); Map internalProperties = new HashMap<>(); - if (catalog instanceof ExternalCatalog) { - internalProperties.put(REMOTE_URL, ((ExternalCatalog) catalog).getRemoteUrl()); - } internalProperties.put(CATALOG_TYPE_PROPERTY, catalog.getType().name()); builder.setInternalProperties(internalProperties); builder.setStorageConfigurationInfo( @@ -118,7 +114,6 @@ public Catalog asCatalog() { : ExternalCatalog.builder() .setType(Catalog.TypeEnum.EXTERNAL) .setName(getName()) - .setRemoteUrl(getInternalPropertiesAsMap().get(REMOTE_URL)) .setProperties(catalogProps) .setCreateTimestamp(getCreateTimestamp()) .setLastUpdateTimestamp(getLastUpdateTimestamp()) diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 54c3b96759..14da2596e9 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -850,9 +850,92 @@ components: - $ref: "#/components/schemas/Catalog" - type: object properties: - remoteUrl: - type: string - description: URL to the remote catalog API + connectionConfigInfo: + $ref: "#/components/schemas/ConnectionConfigInfo" + + ConnectionConfigInfo: + type: object + description: A connection configuration representing a remote catalog service + properties: + connectionType: + type: string + enum: + - ICEBERG_REST + description: The type of remote catalog service represented by this connection + remoteUri: + type: string + description: URI to the remote catalog service + required: + - connectionType + discriminator: + propertyName: connectionType + mapping: + ICEBERG_REST: "#/components/schemas/IcebergRestConnectionConfigInfo" + + IcebergRestConnectionConfigInfo: + type: object + description: Configuration necessary for connecting to an Iceberg REST Catalog + allOf: + - $ref: '#/components/schemas/ConnectionConfigInfo' + properties: + remoteCatalogName: + type: string + description: The name of a remote catalog instance within the remote catalog service; in some older systems + this is specified as the 'warehouse' when multiple logical catalogs are served under the same base + remoteUri, and often translates into a 'prefix' added to all REST resource paths + restAuthentication: + $ref: "#/components/schemas/RestAuthenticationInfo" + + RestAuthenticationInfo: + type: object + description: Authentication-specific information for a REST connection + properties: + restAuthenticationType: + type: string + enum: + - OAUTH + - BEARER + description: The type of authentication to use when connecting to the remote rest service + required: + - restAuthenticationType + discriminator: + propertyName: restAuthenticationType + mapping: + OAUTH: "#/components/schemas/OauthRestAuthenticationInfo" + BEARER: "#/components/schemas/BearerRestAuthenticationInfo" + + OauthRestAuthenticationInfo: + type: object + description: OAuth authentication based on client_id/client_secret + allOf: + - $ref: '#/components/schemas/RestAuthenticationInfo' + properties: + tokenUri: + type: string + description: Token server URI + clientId: + type: string + description: oauth client id + clientSecret: + type: string + format: password + description: oauth client secret (input-only) + scopes: + type: array + items: + type: string + description: oauth scopes to specify when exchanging for a short-lived access token + + BearerRestAuthenticationInfo: + type: object + description: Bearer authentication directly embedded in request auth headers + allOf: + - $ref: '#/components/schemas/RestAuthenticationInfo' + properties: + bearerToken: + type: string + format: password + description: Bearer token (input-only) StorageConfigInfo: type: object From 1d16f046743b4019634ae85062f4ba1552d9b6a4 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 18 Mar 2025 04:36:39 +0000 Subject: [PATCH 2/5] Rename types per PR suggestions; OAuthClientCredentialsParameters, BearerAuthenticationParameters. Also rename 'remoteUri' to just 'uri' --- spec/polaris-management-service.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index ad00b12c7f..6b621ac61b 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -862,7 +862,7 @@ components: enum: - ICEBERG_REST description: The type of remote catalog service represented by this connection - remoteUri: + uri: type: string description: URI to the remote catalog service required: @@ -882,11 +882,11 @@ components: type: string description: The name of a remote catalog instance within the remote catalog service; in some older systems this is specified as the 'warehouse' when multiple logical catalogs are served under the same base - remoteUri, and often translates into a 'prefix' added to all REST resource paths + uri, and often translates into a 'prefix' added to all REST resource paths restAuthentication: - $ref: "#/components/schemas/RestAuthenticationInfo" + $ref: "#/components/schemas/AuthenticationParameters" - RestAuthenticationInfo: + AuthenticationParameters: type: object description: Authentication-specific information for a REST connection properties: @@ -901,14 +901,14 @@ components: discriminator: propertyName: restAuthenticationType mapping: - OAUTH: "#/components/schemas/OauthRestAuthenticationInfo" - BEARER: "#/components/schemas/BearerRestAuthenticationInfo" + OAUTH: "#/components/schemas/OAuthClientCredentialsParameters" + BEARER: "#/components/schemas/BearerAuthenticationParameters" - OauthRestAuthenticationInfo: + OAuthClientCredentialsParameters: type: object description: OAuth authentication based on client_id/client_secret allOf: - - $ref: '#/components/schemas/RestAuthenticationInfo' + - $ref: '#/components/schemas/AuthenticationParameters' properties: tokenUri: type: string @@ -926,11 +926,11 @@ components: type: string description: oauth scopes to specify when exchanging for a short-lived access token - BearerRestAuthenticationInfo: + BearerAuthenticationParameters: type: object description: Bearer authentication directly embedded in request auth headers allOf: - - $ref: '#/components/schemas/RestAuthenticationInfo' + - $ref: '#/components/schemas/AuthenticationParameters' properties: bearerToken: type: string From 42fa76b9196113b2c8d541b8316f439067ba6443 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Thu, 20 Mar 2025 03:14:47 +0000 Subject: [PATCH 3/5] Update some additional variable names to better match the type renames. --- spec/polaris-management-service.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 6b621ac61b..d9c0c5c04d 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -883,23 +883,23 @@ components: description: The name of a remote catalog instance within the remote catalog service; in some older systems this is specified as the 'warehouse' when multiple logical catalogs are served under the same base uri, and often translates into a 'prefix' added to all REST resource paths - restAuthentication: + authenticationParameters: $ref: "#/components/schemas/AuthenticationParameters" AuthenticationParameters: type: object description: Authentication-specific information for a REST connection properties: - restAuthenticationType: + authenticationType: type: string enum: - OAUTH - BEARER description: The type of authentication to use when connecting to the remote rest service required: - - restAuthenticationType + - authenticationType discriminator: - propertyName: restAuthenticationType + propertyName: authenticationType mapping: OAUTH: "#/components/schemas/OAuthClientCredentialsParameters" BEARER: "#/components/schemas/BearerAuthenticationParameters" From dadf5b385f6e32fc5ece073abc622d4c1c426708 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Fri, 21 Mar 2025 02:19:47 +0000 Subject: [PATCH 4/5] Add a note in the description of ConnectionConfigInfo that it's an experimental API --- spec/polaris-management-service.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index d9c0c5c04d..7d114cdc89 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -855,7 +855,8 @@ components: ConnectionConfigInfo: type: object - description: A connection configuration representing a remote catalog service + description: A connection configuration representing a remote catalog service. IMPORTANT - Specifying a + ConnectionConfigInfo in an ExternalCatalog is currently an experimental API and is subject to change. properties: connectionType: type: string From 7f3c4f5db2e3e52b2931fe43d73130179f0b8f14 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 25 Mar 2025 04:03:34 +0000 Subject: [PATCH 5/5] Now that AuthenticationParameters isn't iceberg-specific, move it to the base ConnectionConfigInfo --- spec/polaris-management-service.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 7d114cdc89..318f17a6c6 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -866,6 +866,8 @@ components: uri: type: string description: URI to the remote catalog service + authenticationParameters: + $ref: "#/components/schemas/AuthenticationParameters" required: - connectionType discriminator: @@ -884,8 +886,6 @@ components: description: The name of a remote catalog instance within the remote catalog service; in some older systems this is specified as the 'warehouse' when multiple logical catalogs are served under the same base uri, and often translates into a 'prefix' added to all REST resource paths - authenticationParameters: - $ref: "#/components/schemas/AuthenticationParameters" AuthenticationParameters: type: object