From 8061211aa47a27f3d95350c8988542e098e56bc0 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 1 Aug 2025 15:15:25 +0200 Subject: [PATCH 01/10] Add support for S3 request signing Fixes #32. --- CHANGELOG.md | 8 + LICENSE | 1 + README.md | 4 +- api/README.md | 2 + api/s3-sign-service/build.gradle.kts | 114 +++++ .../s3/sign/model/PolarisS3SignRequest.java | 53 ++ .../s3/sign/model/PolarisS3SignResponse.java | 36 ++ bom/build.gradle.kts | 1 + gradle/projects.main.properties | 1 + ...PolarisS3RemoteSigningIntegrationTest.java | 156 ++++++ .../auth/PolarisAuthorizableOperation.java | 6 +- .../core/auth/PolarisAuthorizerImpl.java | 7 +- .../core/config/FeatureConfiguration.java | 10 + .../polaris/core/entity/PolarisPrivilege.java | 1 + .../polaris/core/rest/PolarisEndpoints.java | 19 + .../core/rest/PolarisResourcePaths.java | 15 + .../polaris/core/storage/AccessConfig.java | 19 +- .../storage/InMemoryStorageIntegration.java | 3 +- .../core/storage/StorageAccessProperty.java | 17 + .../aws/AwsCredentialsStorageIntegration.java | 30 +- .../AzureCredentialsStorageIntegration.java | 3 +- .../gcp/GcpCredentialsStorageIntegration.java | 3 +- .../core/entity/PolarisPrivilegeTest.java | 3 +- .../core/storage/AccessConfigTest.java | 15 +- .../cache/StorageCredentialCacheTest.java | 8 + runtime/service/build.gradle.kts | 1 + .../service/it/S3RemoteSigningMinIOIT.java | 90 ++++ .../catalog/common/CatalogHandler.java | 41 ++ .../catalog/iceberg/IcebergCatalog.java | 2 +- .../iceberg/IcebergCatalogAdapter.java | 29 +- .../iceberg/IcebergCatalogHandler.java | 101 ++-- .../catalog/io/AccessConfigProvider.java | 81 +++- .../service/catalog/io/FileIOUtil.java | 7 +- ...PolarisStorageIntegrationProviderImpl.java | 5 +- .../service/storage/StorageConfiguration.java | 31 +- .../sign/S3RemoteSigningCatalogAdapter.java | 108 +++++ .../sign/S3RemoteSigningCatalogHandler.java | 198 ++++++++ .../storage/s3/sign/S3RequestSigner.java | 30 ++ .../storage/s3/sign/S3RequestSignerImpl.java | 89 ++++ .../service/task/TaskFileIOSupplier.java | 2 +- .../service/admin/PolarisAuthzTestBase.java | 5 +- ...bstractPolarisGenericTableCatalogTest.java | 10 +- .../iceberg/AbstractIcebergCatalogTest.java | 10 +- .../AbstractIcebergCatalogViewTest.java | 12 +- .../IcebergCatalogHandlerAuthzTest.java | 118 ++++- .../policy/AbstractPolicyCatalogTest.java | 10 +- .../it/RestCatalogMinIOSpecialTest.java | 453 ++++++++++++++++++ .../S3RemoteSigningMinIOIntegrationTest.java | 90 ++++ .../storage/StorageConfigurationTest.java | 4 +- ...3RemoteSigningCatalogHandlerAuthzTest.java | 127 +++++ .../s3/sign/S3RequestSignerImplTest.java | 223 +++++++++ .../apache/polaris/service/TestServices.java | 12 +- .../managing-security/access-control.md | 25 +- spec/README.md | 42 +- spec/s3-sign/iceberg-s3-signer-open-api.yaml | 154 ++++++ spec/s3-sign/polaris-s3-sign-service.yaml | 108 +++++ 56 files changed, 2622 insertions(+), 131 deletions(-) create mode 100644 api/s3-sign-service/build.gradle.kts create mode 100644 api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java create mode 100644 api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java create mode 100644 runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java create mode 100644 runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java create mode 100644 spec/s3-sign/iceberg-s3-signer-open-api.yaml create mode 100644 spec/s3-sign/polaris-s3-sign-service.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 493e0c60db..a3de251340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti [Iceberg Metrics Reporting]: https://iceberg.apache.org/docs/latest/metrics-reporting/ +- **S3 remote request signing** has been added, allowing Polaris to work with S3-compatible object storage systems. + *Remote signing is currently experimental and not enabled by default*. In particular, RBAC checks are currently not + production-ready. One new table privilege was introduced: `TABLE_REMOTE_SIGN`. To enable remote signing: + 1. Set the system-wide property `REMOTE_SIGNING_ENABLED` or the catalog-level `polaris.request-signing.enabled` + property to `true`. + 2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The role must also be granted the `TABLE_READ_DATA` + and `TABLE_WRITE_DATA` privileges. + ### Upgrade notes - The legacy management endpoints at `/metrics` and `/healthcheck` have been removed. Please use the diff --git a/LICENSE b/LICENSE index 8f50538f8d..2104ef28c6 100644 --- a/LICENSE +++ b/LICENSE @@ -217,6 +217,7 @@ This product includes code from Apache Iceberg. * spec/iceberg-rest-catalog-open-api.yaml * spec/polaris-catalog-apis/oauth-tokens-api.yaml +* spec/s3-sign/iceberg-s3-signer-open-api.yaml * integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java diff --git a/README.md b/README.md index 3245a4b086..fd6a1fb762 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ Apache Polaris is organized into the following modules: - `polaris-api-management-service` - Polaris Management API service classes - `polaris-api-iceberg-service` - The Iceberg REST service classes - `polaris-api-catalog-service` - The Polaris Catalog API service classes - - Runtime modules: + - `polaris-api-s3-sign-service` - The Iceberg REST service for S3 remote signing +- Runtime modules: - [`polaris-admin`](./runtime/admin/README.md) - The Polaris Admin Tool; mainly for bootstrapping persistence - [`polaris-runtime-defaults`](./runtime/defaults/README.md) - The runtime configuration defaults - [`polaris-distribution`](./runtime/distribution/README.md) - The Polaris distribution @@ -97,7 +98,6 @@ In addition to modules, there are: Outside of this repository, there are several other tools that can be found in a separate [Polaris-Tools](https://github.com/apache/polaris-tools) repository. ## Building and Running - Apache Polaris is built using Gradle with Java 21+ and Docker 27+. - `./gradlew build` - To build and run tests. Make sure Docker is running, as the integration tests depend on it. diff --git a/api/README.md b/api/README.md index 7c7fb61fc0..d8093d99be 100644 --- a/api/README.md +++ b/api/README.md @@ -33,6 +33,8 @@ This directory contains the API modules for Apache Polaris. Iceberg REST API. - [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris native Catalog REST API. +- [`polaris-api-s3-sign-service`](s3-sign-service): contains the model and service classes + for the S3 remote signing REST API. The classes in these modules are generated from the OpenAPI specification files in the [`spec`](../spec) directory. \ No newline at end of file diff --git a/api/s3-sign-service/build.gradle.kts b/api/s3-sign-service/build.gradle.kts new file mode 100644 index 0000000000..c3efbb14bb --- /dev/null +++ b/api/s3-sign-service/build.gradle.kts @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + alias(libs.plugins.openapi.generator) + id("polaris-client") + id("org.kordamp.gradle.jandex") +} + +dependencies { + implementation(project(":polaris-core")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + implementation("org.apache.iceberg:iceberg-aws") + + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.swagger.annotations) + + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.ws.rs.api) + + implementation(platform(libs.micrometer.bom)) + implementation("io.micrometer:micrometer-core") + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + + compileOnly(libs.microprofile.fault.tolerance.api) + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) +} + +val rootDir = rootProject.layout.projectDirectory +val specsDir = rootDir.dir("spec") +val templatesDir = rootDir.dir("server-templates") +// Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the +// whole directory to its task output, but 'generated/' is not exclusive to that task and in turn +// breaks Gradle's caching. +val generatedDir = project.layout.buildDirectory.dir("generated-openapi") +val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java") + +openApiGenerate { + // The OpenAPI generator does NOT resolve relative paths correctly against the Gradle project + // directory + inputSpec = provider { specsDir.file("s3-sign/polaris-s3-sign-service.yaml").asFile.absolutePath } + generatorName = "jaxrs-resteasy" + outputDir = provider { generatedDir.get().asFile.absolutePath } + apiPackage = "org.apache.polaris.service.s3.sign.api" + ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath }) + templateDir.set(provider { templatesDir.asFile.absolutePath }) + removeOperationIdPrefix.set(true) + globalProperties.put("apis", "S3SignerApi") + globalProperties.put("models", "false") + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("resourceName", "catalog") + configOptions.put("useTags", "true") + configOptions.put("useBeanValidation", "false") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("hideGenerationTimestamp", "true") + additionalProperties.put("apiNamePrefix", "IcebergRest") + additionalProperties.put("apiNameSuffix", "") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables.put("basePath", "api/s3-sign") + modelNameMappings = mapOf("S3SignRequest" to "PolarisS3SignRequest") + typeMappings = + mapOf("S3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest") + importMappings = + mapOf( + "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse", + "PolarisS3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest", + "SignS3Request200Response" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse", + ) +} + +listOf("sourcesJar", "compileJava", "processResources").forEach { task -> + tasks.named(task) { dependsOn("openApiGenerate") } +} + +sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } } + +tasks.named("openApiGenerate") { + inputs.dir(templatesDir) + inputs.dir(specsDir) + actions.addFirst { delete { delete(generatedDir) } } +} + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java new file mode 100644 index 0000000000..efc9d63746 --- /dev/null +++ b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.s3.sign.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import jakarta.annotation.Nullable; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.immutables.PolarisImmutable; +import org.immutables.value.Value; + +/** + * Request for S3 signing. + * + *

Copy of {@link S3SignRequest}, because the original does not have Jackson annotations. + */ +@PolarisImmutable +@JsonDeserialize(as = ImmutablePolarisS3SignRequest.class) +@JsonSerialize(as = ImmutablePolarisS3SignRequest.class) +@SuppressWarnings("immutables:subtype") +public interface PolarisS3SignRequest extends S3SignRequest { + + @Value.Default + @Nullable // Replace javax.annotation.Nullable from S3SignRequest with jakarta.annotation.Nullable + @Override + default String body() { + return null; + } + + default boolean write() { + return method().equalsIgnoreCase("PUT") + || method().equalsIgnoreCase("POST") + || method().equalsIgnoreCase("DELETE") + || method().equalsIgnoreCase("PATCH"); + } +} diff --git a/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java new file mode 100644 index 0000000000..9f4cb778cf --- /dev/null +++ b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.s3.sign.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Response for S3 signing requests. + * + *

Copy of {@link S3SignResponse}, because the original does not have Jackson annotations. + */ +@PolarisImmutable +@JsonDeserialize(as = ImmutablePolarisS3SignResponse.class) +@JsonSerialize(as = ImmutablePolarisS3SignResponse.class) +@SuppressWarnings("immutables:subtype") +public interface PolarisS3SignResponse extends S3SignResponse {} diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index d590052228..ccf8eb5698 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { api(project(":polaris-api-iceberg-service")) api(project(":polaris-api-management-model")) api(project(":polaris-api-management-service")) + api(project(":polaris-api-s3-sign-service")) api(project(":polaris-container-spec-helper")) api(project(":polaris-minio-testcontainer")) diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 7c4fd61e53..4efb07373b 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -24,6 +24,7 @@ polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service polaris-api-catalog-service=api/polaris-catalog-service +polaris-api-s3-sign-service=api/s3-sign-service polaris-runtime-defaults=runtime/defaults polaris-runtime-service=runtime/service polaris-server=runtime/server diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java new file mode 100644 index 0000000000..7044190f56 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_ENDPOINT; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_PATH_STYLE_ACCESS; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.service.it.env.CatalogConfig; +import org.apache.polaris.service.it.env.RestCatalogConfig; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +/** Integration tests for S3 remote signing. */ +@CatalogConfig(properties = {"header.X-Iceberg-Access-Delegation", "remote-signing"}) +@RestCatalogConfig({ + // The default client file IO implementation is InMemoryFileIO, + // which does not support remote signing. + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.io.ResolvingFileIO", +}) +public abstract class PolarisS3RemoteSigningIntegrationTest + extends PolarisRestCatalogIntegrationBase { + + @Override + protected StorageConfigInfo getStorageConfigInfo() { + return AwsStorageConfigInfo.builder() + .setRoleArn(roleArn()) + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setPathStyleAccess(pathStyleAccess()) + .setAllowedLocations(allowedLocations()) + .setEndpoint(endpoint().orElse(null)) + .setStsUnavailable(stsUnavailable()) + .build(); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + ImmutableMap.Builder builder = + super.clientFileIOProperties() + .put(AWS_PATH_STYLE_ACCESS.getPropertyName(), String.valueOf(pathStyleAccess())); + endpoint().ifPresent(endpoint -> builder.put(AWS_ENDPOINT.getPropertyName(), endpoint)); + return builder; + } + + protected String roleArn() { + return "arn:aws:iam::123456789012:role/my-role"; + } + + protected boolean pathStyleAccess() { + return true; + } + + protected Optional endpoint() { + return Optional.empty(); + } + + protected abstract List allowedLocations(); + + protected boolean stsUnavailable() { + return false; + } + + @CatalogConfig(properties = {"polaris.config.remote-signing.enabled", "false"}) + @Test + public void testInternalCatalogRemoteSigningDisabled() { + @SuppressWarnings("resource") + RESTCatalog catalog = catalog(); + Namespace ns1 = Namespace.of("ns1"); + catalog.createNamespace(ns1); + TableIdentifier tableIdentifier = TableIdentifier.of(ns1, "my_table"); + assertThatThrownBy(() -> catalog.createTable(tableIdentifier, SCHEMA)) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Remote signing is not enabled for this catalog") + .hasMessageContaining(FeatureConfiguration.REMOTE_SIGNING_ENABLED.key()) + .hasMessageContaining(FeatureConfiguration.REMOTE_SIGNING_ENABLED.catalogConfig()); + } + + @CatalogConfig(Catalog.TypeEnum.EXTERNAL) + @Test + public void testExternalCatalogRemoteSigningDisabled() { + @SuppressWarnings("resource") + RESTCatalog catalog = catalog(); + Namespace ns1 = Namespace.of("ns1"); + catalog.createNamespace(ns1); + TableMetadata tableMetadata = + TableMetadata.newTableMetadata( + SCHEMA, + PartitionSpec.unpartitioned(), + externalCatalogBaseLocation() + "/ns1/my_table", + Map.of()); + try (ResolvingFileIO resolvingFileIO = initializeClientFileIO(new ResolvingFileIO())) { + String fileLocation = + externalCatalogBaseLocation() + "/ns1/my_table/metadata/v1.metadata.json"; + TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); + catalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); + try { + assertThatThrownBy(() -> catalog.loadTable(TableIdentifier.of(ns1, "my_table"))) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Remote signing is not enabled for external catalogs"); + } finally { + resolvingFileIO.deleteFile(fileLocation); + } + } + } + + @Test + @Override + @Disabled("It's not possible to request an access delegation mode when registering a table.") + public void testRegisterTable() { + // FIXME this test should work if Polaris could send the right AccessConfig even if no + // delegation mode was requested when registering the table. + } + + @Test + @Override + @Disabled("This test is explicitly for vended credentials") + public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisabled() {} + + @Test + @Override + @Disabled("This test is explicitly for vended credentials") + public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisabled() {} +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java index 9d12cc148a..af642fe3d7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java @@ -85,6 +85,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOTE_SIGN; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_SNAPSHOTS; @@ -246,7 +247,10 @@ public enum PolarisAuthorizableOperation { REMOVE_TABLE_PROPERTIES(TABLE_REMOVE_PROPERTIES), SET_TABLE_STATISTICS(TABLE_SET_STATISTICS), REMOVE_TABLE_STATISTICS(TABLE_REMOVE_STATISTICS), - REMOVE_TABLE_PARTITION_SPECS(TABLE_REMOVE_PARTITION_SPECS); + REMOVE_TABLE_PARTITION_SPECS(TABLE_REMOVE_PARTITION_SPECS), + SIGN_S3_READ_REQUEST(EnumSet.of(TABLE_REMOTE_SIGN, TABLE_READ_DATA)), + SIGN_S3_WRITE_REQUEST(EnumSet.of(TABLE_REMOTE_SIGN, TABLE_WRITE_DATA)), + ; private final EnumSet privilegesOnTarget; private final EnumSet privilegesOnSecondary; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java index 489bc4fdae..68185b05e5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java @@ -99,6 +99,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_STRUCTURE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOTE_SIGN; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_SNAPSHOTS; @@ -144,7 +145,7 @@ import org.slf4j.LoggerFactory; /** - * Performs hierarchical resolution logic by matching the transively expanded set of grants to a + * Performs hierarchical resolution logic by matching the transitively expanded set of grants to a * calling principal against the cascading permissions over the parent hierarchy of a target * Securable. * @@ -461,6 +462,10 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer { SUPER_PRIVILEGES.putAll( VIEW_FULL_METADATA, List.of(CATALOG_MANAGE_CONTENT, CATALOG_MANAGE_METADATA, VIEW_FULL_METADATA)); + SUPER_PRIVILEGES.putAll( + TABLE_REMOTE_SIGN, + List.of( + CATALOG_MANAGE_CONTENT, CATALOG_MANAGE_METADATA, TABLE_CREATE, TABLE_FULL_METADATA)); // Catalog privileges SUPER_PRIVILEGES.putAll( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 1772f47256..4c90feeb8d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -438,4 +438,14 @@ public static void enforceFeatureEnabledOrThrow( "If set to true (default), allow credential vending for external catalogs. Note this requires ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING to be true first.") .defaultValue(true) .buildFeatureConfiguration(); + + public static final FeatureConfiguration REMOTE_SIGNING_ENABLED = + PolarisConfiguration.builder() + .key("REMOTE_SIGNING_ENABLED") + .catalogConfig("polaris.config.remote-signing.enabled") + .description( + "If true, the remote signing endpoints are enabled either globally, or for a specific catalog. " + + "This feature is currently experimental and may change in future releases.") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java index d76a6d457a..52f4a52f66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java @@ -245,6 +245,7 @@ public enum PolarisPrivilege { PolarisEntityType.TABLE_LIKE, List.of(PolarisEntitySubType.ICEBERG_TABLE, PolarisEntitySubType.GENERIC_TABLE), PolarisEntityType.CATALOG_ROLE), + TABLE_REMOTE_SIGN(103, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE), ; /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java index 2bae38ff7e..5b8f85e0d0 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java @@ -23,6 +23,7 @@ import org.apache.iceberg.rest.Endpoint; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.entity.CatalogEntity; public class PolarisEndpoints { // Generic table endpoints @@ -73,6 +74,12 @@ public class PolarisEndpoints { .add(V1_GET_APPLICABLE_POLICIES) .build(); + public static final Endpoint V1_S3_REMOTE_SIGNING = + Endpoint.create("POST", PolarisResourcePaths.V1_S3_REMOTE_SIGNING); + + public static final Set REMOTE_SIGNING_ENDPOINTS = + ImmutableSet.builder().add(V1_S3_REMOTE_SIGNING).build(); + /** * Get the generic table endpoints. Returns GENERIC_TABLE_ENDPOINTS if ENABLE_GENERIC_TABLES is * set to true, otherwise, returns an empty set. @@ -92,4 +99,16 @@ public static Set getSupportedPolicyEndpoints(RealmConfig realmConfig) boolean policyStoreEnabled = realmConfig.getConfig(FeatureConfiguration.ENABLE_POLICY_STORE); return policyStoreEnabled ? POLICY_STORE_ENDPOINTS : ImmutableSet.of(); } + + /** + * Get the remote signing endpoints. Returns {@link #REMOTE_SIGNING_ENDPOINTS} if {@link + * FeatureConfiguration#REMOTE_SIGNING_ENABLED} is set globally to true or if the catalog enables + * remote signing; otherwise, returns an empty set. + */ + public static Set getSupportedRemoteSigningEndpoints( + RealmConfig realmConfig, CatalogEntity catalogEntity) { + boolean remoteSigningEnabled = + realmConfig.getConfig(FeatureConfiguration.REMOTE_SIGNING_ENABLED, catalogEntity); + return remoteSigningEnabled ? REMOTE_SIGNING_ENDPOINTS : ImmutableSet.of(); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java index 16eea08da2..bea8544bf1 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java @@ -42,6 +42,10 @@ public class PolarisResourcePaths { "/polaris/v1/{prefix}/namespaces/{namespace}/policies/{policy-name}/mappings"; public static final String V1_APPLICABLE_POLICIES = "/polaris/v1/{prefix}/applicable-policies"; + // S3 Remote Signing endpoint + public static final String V1_S3_REMOTE_SIGNING = + "/s3-sign/v1/{prefix}/namespaces/{namespace}/tables/{table}"; + private final String prefix; public PolarisResourcePaths(String prefix) { @@ -78,4 +82,15 @@ public String genericTable(TableIdentifier ident) { "generic-tables", RESTUtil.encodeString(ident.name())); } + + public String s3RemoteSigning(TableIdentifier ident) { + return SLASH.join( + "s3-sign", + "v1", + prefix, + "namespaces", + RESTUtil.encodeNamespace(ident.namespace()), + "tables", + RESTUtil.encodeString(ident.name())); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java index 94e74a3d66..8afe389314 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/AccessConfig.java @@ -23,10 +23,13 @@ import java.util.Map; import java.util.Optional; import org.apache.polaris.immutables.PolarisImmutable; -import org.immutables.value.Value; @PolarisImmutable public interface AccessConfig { + + AccessConfig EMPTY = + AccessConfig.builder().supportsCredentialVending(false).supportsRemoteSigning(false).build(); + Map credentials(); Map extraProperties(); @@ -43,10 +46,13 @@ public interface AccessConfig { * Indicates whether the storage integration subsystem that produced this object is capable of * credential vending in principle. */ - @Value.Default - default boolean supportsCredentialVending() { - return true; - } + boolean supportsCredentialVending(); + + /** + * Indicates whether the storage integration subsystem that produced this object is capable of + * remote signing in principle. + */ + boolean supportsRemoteSigning(); default String get(StorageAccessProperty key) { if (key.isCredential()) { @@ -77,6 +83,9 @@ interface Builder { @CanIgnoreReturnValue Builder supportsCredentialVending(boolean supportsCredentialVending); + @CanIgnoreReturnValue + Builder supportsRemoteSigning(boolean supportsRemoteSigning); + default Builder put(StorageAccessProperty key, String value) { if (key.isExpirationTimestamp()) { expiresAt(Instant.ofEpochMilli(Long.parseLong(value))); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java index ea4ca1c594..71b25e9118 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.storage; import jakarta.annotation.Nonnull; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,7 +56,7 @@ protected InMemoryStorageIntegration(T config, String identifierOrId) { */ public static Map> validateAllowedLocations( @Nonnull RealmConfig realmConfig, - @Nonnull List allowedLocationsToValid, + @Nonnull Collection allowedLocationsToValid, @Nonnull Set actions, @Nonnull Set locations) { // trim trailing / from allowed locations so that locations missing the trailing slash still diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java index 31a92c4f45..0ccc3ce0d6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java @@ -19,6 +19,8 @@ package org.apache.polaris.core.storage; import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; import org.apache.iceberg.azure.AzureProperties; import org.apache.iceberg.gcp.GCPProperties; @@ -52,6 +54,21 @@ public enum StorageAccessProperty { "the endpoint to load vended credentials for a table from the catalog", false, false), + AWS_REMOTE_SIGNING_ENABLED( + Boolean.class, + S3FileIOProperties.REMOTE_SIGNING_ENABLED, + "whether to enable remote signing for S3 requests", + false), + AWS_REMOTE_SIGNER_URI( + String.class, + S3V4RestSignerClient.S3_SIGNER_URI, + "the base URI for the remote signer service, used for signing S3 requests", + false), + AWS_REMOTE_SIGNER_ENDPOINT( + String.class, + S3V4RestSignerClient.S3_SIGNER_ENDPOINT, + "the endpoint for the remote signer service, used for signing S3 requests", + false), GCS_ACCESS_TOKEN(String.class, "gcs.oauth2.token", "the gcs scoped access token", true), GCS_ACCESS_TOKEN_EXPIRES_AT( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 8023f7a607..72a393ef6b 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -80,7 +80,8 @@ public AccessConfig getSubscopedCreds( realmConfig.getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS); AwsStorageConfigurationInfo storageConfig = config(); String region = storageConfig.getRegion(); - AccessConfig.Builder accessConfig = AccessConfig.builder(); + AccessConfig.Builder accessConfig = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); if (shouldUseSts(storageConfig)) { AssumeRoleRequest.Builder request = @@ -120,6 +121,31 @@ public AccessConfig getSubscopedCreds( }); } + addCommonProperties(region, accessConfig, storageConfig, refreshCredentialsEndpoint); + + return accessConfig.build(); + } + + public AccessConfig getRemoteSigningAccessConfig(URI signerUri, String signerEndpoint) { + + AwsStorageConfigurationInfo storageConfig = config(); + AccessConfig.Builder accessConfig = + AccessConfig.builder().supportsCredentialVending(false).supportsRemoteSigning(true); + + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNING_ENABLED, Boolean.TRUE.toString()); + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNER_URI, signerUri.toString()); + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNER_ENDPOINT, signerEndpoint); + + addCommonProperties(storageConfig.getRegion(), accessConfig, storageConfig, Optional.empty()); + + return accessConfig.build(); + } + + private static void addCommonProperties( + String region, + AccessConfig.Builder accessConfig, + AwsStorageConfigurationInfo storageConfig, + Optional refreshCredentialsEndpoint) { if (region != null) { accessConfig.put(StorageAccessProperty.CLIENT_REGION, region); } @@ -148,8 +174,6 @@ public AccessConfig getSubscopedCreds( String.format( "AWS region must be set when using partition %s", storageConfig.getAwsPartition())); } - - return accessConfig.build(); } private boolean shouldUseSts(AwsStorageConfigurationInfo storageConfig) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index a043a7daa5..e8b1953116 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -181,7 +181,8 @@ static AccessConfig toAccessConfig( String storageDnsName, Instant expiresAt, Optional refreshCredentialsEndpoint) { - AccessConfig.Builder accessConfig = AccessConfig.builder(); + AccessConfig.Builder accessConfig = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); handleAzureCredential(accessConfig, sasToken, storageDnsName, expiresAt); accessConfig.put( StorageAccessProperty.EXPIRATION_TIME, String.valueOf(expiresAt.toEpochMilli())); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index c0568cc9b5..c56e1fa0cc 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -109,7 +109,8 @@ public AccessConfig getSubscopedCreds( // If expires_in missing, use source credential's expire time, which require another api call to // get. - AccessConfig.Builder accessConfig = AccessConfig.builder(); + AccessConfig.Builder accessConfig = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); accessConfig.put(StorageAccessProperty.GCS_ACCESS_TOKEN, token.getTokenValue()); accessConfig.put( StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java index 14596911fd..3355e23c88 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java @@ -131,7 +131,8 @@ static Stream polarisPrivileges() { Arguments.of(100, PolarisPrivilege.TABLE_REMOVE_STATISTICS), Arguments.of(101, PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS), Arguments.of(102, PolarisPrivilege.TABLE_MANAGE_STRUCTURE), - Arguments.of(103, null)); + Arguments.of(103, PolarisPrivilege.TABLE_REMOTE_SIGN), + Arguments.of(104, null)); } @ParameterizedTest diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java index 57e1f14650..54dc3b8b56 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/AccessConfigTest.java @@ -34,7 +34,8 @@ public class AccessConfigTest { @Test public void testPutGet() { - AccessConfig.Builder b = AccessConfig.builder(); + AccessConfig.Builder b = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.put(AWS_ENDPOINT, "ep1"); b.put(AWS_SECRET_KEY, "sk2"); AccessConfig c = b.build(); @@ -46,7 +47,8 @@ public void testPutGet() { @Test public void testGetExtraProperty() { - AccessConfig.Builder b = AccessConfig.builder(); + AccessConfig.Builder b = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra"); AccessConfig c = b.build(); assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(), "extra")); @@ -55,7 +57,8 @@ public void testGetExtraProperty() { @Test public void testGetInternalProperty() { - AccessConfig.Builder b = AccessConfig.builder(); + AccessConfig.Builder b = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra"); b.putInternalProperty(AWS_ENDPOINT.getPropertyName(), "ep1"); AccessConfig c = b.build(); @@ -66,7 +69,8 @@ public void testGetInternalProperty() { @Test public void testNoCredentialOverride() { - AccessConfig.Builder b = AccessConfig.builder(); + AccessConfig.Builder b = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.put(AWS_SECRET_KEY, "sk-test"); b.putExtraProperty(AWS_SECRET_KEY.getPropertyName(), "sk-extra"); b.putInternalProperty(AWS_SECRET_KEY.getPropertyName(), "sk-internal"); @@ -79,7 +83,8 @@ public void testNoCredentialOverride() { @Test public void testExpiresAt() { - AccessConfig.Builder b = AccessConfig.builder(); + AccessConfig.Builder b = + AccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); assertThat(b.build().expiresAt()).isEmpty(); b.put(GCS_ACCESS_TOKEN_EXPIRES_AT, "111"); assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(111)); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index a51badf4b8..0e9e4c6124 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -413,6 +413,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( AccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AWS_KEY_ID, "key_id_" + finalI) .put(StorageAccessProperty.AWS_SECRET_KEY, "key_secret_" + finalI) .put(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, expireTime) @@ -422,6 +424,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( AccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_" + finalI) .put(StorageAccessProperty.EXPIRATION_TIME, expireTime) .build())); @@ -429,6 +433,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( AccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.GCS_ACCESS_TOKEN, "gcs_token_" + finalI) .put(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, expireTime) .build())); @@ -460,6 +466,8 @@ public void testExtraProperties() { ScopedCredentialsResult properties = new ScopedCredentialsResult( AccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AWS_SECRET_KEY, "super-secret-123") .put(StorageAccessProperty.AWS_ENDPOINT, "test-endpoint1") .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, "true") diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index c0f462982a..dda9c93680 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { implementation(project(":polaris-api-management-service")) implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-api-catalog-service")) + implementation(project(":polaris-api-s3-sign-service")) runtimeOnly(project(":polaris-relational-jdbc")) diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java new file mode 100644 index 0000000000..6c0f497dcf --- /dev/null +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.service.it.test.PolarisS3RemoteSigningIntegrationTest; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusIntegrationTest +@TestProfile(S3RemoteSigningMinIOIT.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class S3RemoteSigningMinIOIT extends PolarisS3RemoteSigningIntegrationTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static URI storageBase; + private static String endpoint; + + @BeforeAll + static void setup( + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess) { + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @Override + protected List allowedLocations() { + return List.of(storageBase.toString()); + } + + @Override + protected Optional endpoint() { + return Optional.of(endpoint); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + return super.clientFileIOProperties() + // Grant direct access to the MinIO bucket; this FileIO instance does not + // use access delegation. + .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) + .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 9d5778ac29..a35b39ebdd 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -394,6 +394,47 @@ protected void authorizeRenameTableLikeOperationOrThrow( initializeCatalog(); } + protected void authorizeRemoteSigningOrThrow( + EnumSet ops, TableIdentifier identifier) { + + ensureResolutionManifestForTable(identifier); + PolarisResolvedPathWrapper target = + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE, true); + + // If the table doesn't exist, we still need to check allowed locations from the parent + // namespace. + if (target == null) { + resolutionManifest = newResolutionManifest(); + Namespace namespace = identifier.namespace(); + resolutionManifest.addPath( + new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), + namespace); + resolutionManifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.tableIdentifierToList(identifier), + PolarisEntityType.TABLE_LIKE, + true /* optional */), + identifier); + resolutionManifest.resolveAll(); + target = resolutionManifest.getResolvedPath(namespace, true); + if (target == null) { + throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); + } + } + + for (PolarisAuthorizableOperation op : ops) { + authorizer.authorizeOrThrow( + polarisPrincipal, + resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), + op, + target, + null /* secondary */); + } + + initializeCatalog(); + } + /** * Helper function for when a TABLE_LIKE entity is not found so we want to throw the appropriate * exception. Used in Iceberg APIs, so the Iceberg messages cannot be changed. diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index 75742412e3..8c67c77d69 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -2079,7 +2079,7 @@ private FileIO loadFileIOForTableLike( Map tableProperties, Set storageActions) { AccessConfig accessConfig = - accessConfigProvider.getAccessConfig( + accessConfigProvider.getAccessConfigForCredentialsVending( callContext, identifier, readLocations, diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index c636fb075c..bdb7286ab2 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog.iceberg; +import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import static org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation.validateIcebergProperties; @@ -35,6 +36,7 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.EnumSet; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -67,7 +69,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.credentials.PolarisCredentialManager; -import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; @@ -346,11 +348,13 @@ public Response updateProperties( catalog -> Response.ok(catalog.updateNamespaceProperties(ns, revisedRequest)).build()); } - private EnumSet parseAccessDelegationModes(String accessDelegationMode) { + private static Set parseAccessDelegationModes(String accessDelegationMode) { EnumSet delegationModes = AccessDelegationMode.fromProtocolValuesList(accessDelegationMode); Preconditions.checkArgument( - delegationModes.isEmpty() || delegationModes.contains(VENDED_CREDENTIALS), + delegationModes.isEmpty() + || delegationModes.contains(VENDED_CREDENTIALS) + || delegationModes.contains(REMOTE_SIGNING), "Unsupported access delegation mode: %s", accessDelegationMode); return delegationModes; @@ -365,8 +369,7 @@ public Response createTable( RealmContext realmContext, SecurityContext securityContext) { validateIcebergProperties(realmConfig, createTableRequest.properties()); - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); + Set delegationModes = parseAccessDelegationModes(accessDelegationMode); Namespace ns = decodeNamespace(namespace); return withCatalog( securityContext, @@ -418,8 +421,7 @@ public Response loadTable( String snapshots, RealmContext realmContext, SecurityContext securityContext) { - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); + Set delegationModes = parseAccessDelegationModes(accessDelegationMode); Namespace ns = decodeNamespace(namespace); TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); @@ -451,9 +453,7 @@ public Response loadTable( } private static Optional getRefreshCredentialsEndpoint( - EnumSet delegationModes, - String prefix, - TableIdentifier tableIdentifier) { + Set delegationModes, String prefix, TableIdentifier tableIdentifier) { return delegationModes.contains(AccessDelegationMode.VENDED_CREDENTIALS) ? Optional.of(new PolarisResourcePaths(prefix).credentialsPath(tableIdentifier)) : Optional.empty(); @@ -621,6 +621,7 @@ public Response loadCredentials( catalog.loadTableWithAccessDelegation( tableIdentifier, "all", + EnumSet.of(VENDED_CREDENTIALS), Optional.of(new PolarisResourcePaths(prefix).credentialsPath(tableIdentifier))); return Response.ok( ImmutableLoadCredentialsResponse.builder() @@ -814,8 +815,9 @@ public Response getConfig( throw new NotFoundException("Unable to find warehouse %s", warehouse); } ResolvedPolarisEntity resolvedReferenceCatalog = resolver.getResolvedReferenceCatalog(); - Map properties = - PolarisEntity.of(resolvedReferenceCatalog.getEntity()).getPropertiesAsMap(); + CatalogEntity catalogEntity = + CatalogEntity.of(Objects.requireNonNull(resolvedReferenceCatalog).getEntity()); + Map properties = catalogEntity.getPropertiesAsMap(); String prefix = prefixParser.catalogNameToPrefix(realmContext, warehouse); return Response.ok( @@ -828,6 +830,9 @@ public Response getConfig( .addAll(VIEW_ENDPOINTS) .addAll(PolarisEndpoints.getSupportedGenericTableEndpoints(realmConfig)) .addAll(PolarisEndpoints.getSupportedPolicyEndpoints(realmConfig)) + .addAll( + PolarisEndpoints.getSupportedRemoteSigningEndpoints( + callContext.getRealmConfig(), catalogEntity)) .build()) .build()) .build(); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index a10c7afe76..07dd97c6d4 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -20,6 +20,7 @@ import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING; import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED; +import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import com.google.common.base.Preconditions; @@ -112,6 +113,7 @@ import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; +import org.apache.polaris.service.storage.s3.sign.S3RemoteSigningCatalogHandler; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -400,15 +402,13 @@ public LoadTableResponse createTableDirect(Namespace namespace, CreateTableReque public LoadTableResponse createTableDirectWithWriteDelegation( Namespace namespace, CreateTableRequest request, + Set delegationModes, Optional refreshCredentialsEndpoint) { - return createTableDirect( - namespace, request, EnumSet.of(VENDED_CREDENTIALS), refreshCredentialsEndpoint); + return createTableDirect(namespace, request, delegationModes, refreshCredentialsEndpoint); } public void authorizeCreateTableDirect( - Namespace namespace, - CreateTableRequest request, - EnumSet delegationModes) { + Namespace namespace, CreateTableRequest request, Set delegationModes) { if (delegationModes.isEmpty()) { TableIdentifier identifier = TableIdentifier.of(namespace, request.name()); authorizeCreateTableLikeUnderNamespaceOperationOrThrow( @@ -423,13 +423,16 @@ public void authorizeCreateTableDirect( if (catalog.isStaticFacade()) { throw new BadRequestException("Cannot create table on static-facade external catalogs."); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } } public LoadTableResponse createTableDirect( Namespace namespace, CreateTableRequest request, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { authorizeCreateTableDirect(namespace, request, delegationModes); @@ -526,15 +529,13 @@ public LoadTableResponse createTableStaged(Namespace namespace, CreateTableReque public LoadTableResponse createTableStagedWithWriteDelegation( Namespace namespace, CreateTableRequest request, + Set delegationModes, Optional refreshCredentialsEndpoint) { - return createTableStaged( - namespace, request, EnumSet.of(VENDED_CREDENTIALS), refreshCredentialsEndpoint); + return createTableStaged(namespace, request, delegationModes, refreshCredentialsEndpoint); } private void authorizeCreateTableStaged( - Namespace namespace, - CreateTableRequest request, - EnumSet delegationModes) { + Namespace namespace, CreateTableRequest request, Set delegationModes) { if (delegationModes.isEmpty()) { authorizeCreateTableLikeUnderNamespaceOperationOrThrow( PolarisAuthorizableOperation.CREATE_TABLE_STAGED, @@ -549,13 +550,16 @@ private void authorizeCreateTableStaged( if (catalog.isStaticFacade()) { throw new BadRequestException("Cannot create table on static-facade external catalogs."); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } } public LoadTableResponse createTableStaged( Namespace namespace, CreateTableRequest request, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { authorizeCreateTableStaged(namespace, request, delegationModes); @@ -662,9 +666,10 @@ public Optional loadTableIfStale( public LoadTableResponse loadTableWithAccessDelegation( TableIdentifier tableIdentifier, String snapshots, + Set delegationModes, Optional refreshCredentialsEndpoint) { return loadTableWithAccessDelegationIfStale( - tableIdentifier, null, snapshots, refreshCredentialsEndpoint) + tableIdentifier, null, snapshots, delegationModes, refreshCredentialsEndpoint) .get(); } @@ -682,17 +687,14 @@ public Optional loadTableWithAccessDelegationIfStale( TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String snapshots, + Set delegationModes, Optional refreshCredentialsEndpoint) { return loadTable( - tableIdentifier, - snapshots, - ifNoneMatch, - EnumSet.of(VENDED_CREDENTIALS), - refreshCredentialsEndpoint); + tableIdentifier, snapshots, ifNoneMatch, delegationModes, refreshCredentialsEndpoint); } private Set authorizeLoadTable( - TableIdentifier tableIdentifier, EnumSet delegationModes) { + TableIdentifier tableIdentifier, Set delegationModes) { if (delegationModes.isEmpty()) { authorizeBasicTableLikeOperationOrThrow( PolarisAuthorizableOperation.LOAD_TABLE, @@ -724,7 +726,10 @@ private Set authorizeLoadTable( read, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } return actionsRequested; } @@ -733,7 +738,7 @@ public Optional loadTable( TableIdentifier tableIdentifier, String snapshots, IfNoneMatch ifNoneMatch, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { Set actionsRequested = @@ -785,7 +790,7 @@ public Optional loadTable( private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredentials( TableIdentifier tableIdentifier, TableMetadata tableMetadata, - EnumSet delegationModes, + Set delegationModes, Set actions, Optional refreshCredentialsEndpoint) { LoadTableResponse.Builder responseBuilder = @@ -799,6 +804,33 @@ private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredential return responseBuilder; } + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + + if (delegationMode == REMOTE_SIGNING) { + + S3RemoteSigningCatalogHandler.throwIfRemoteSigningNotEnabled( + callContext.getRealmConfig(), getResolvedCatalogEntity()); + + AccessConfig accessConfig = + accessConfigProvider.getAccessConfigForRemoteSigning( + callContext, catalogName, tableIdentifier, resolvedStoragePath); + + Map credentialConfig = accessConfig.credentials(); + + if (!credentialConfig.isEmpty()) { + responseBuilder.addAllConfig(credentialConfig); + responseBuilder.addCredential( + ImmutableCredential.builder() + .prefix(tableMetadata.location()) + .config(credentialConfig) + .build()); + } + + responseBuilder.addAllConfig(accessConfig.extraProperties()); + + return responseBuilder; + } + if (baseCatalog instanceof IcebergCatalog || realmConfig.getConfig( ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { @@ -811,7 +843,7 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { } AccessConfig accessConfig = - accessConfigProvider.getAccessConfig( + accessConfigProvider.getAccessConfigForCredentialsVending( callContext, tableIdentifier, tableLocations, @@ -819,6 +851,9 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { refreshCredentialsEndpoint, resolvedStoragePath); Map credentialConfig = accessConfig.credentials(); + + // FIXME it would be cleaner to move this into the accessConfigProvider + // or create another method getAccessConfigNoDelegation() if (delegationModes.contains(VENDED_CREDENTIALS)) { if (!credentialConfig.isEmpty()) { responseBuilder.addAllConfig(credentialConfig); @@ -842,6 +877,15 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { return responseBuilder; } + private static AccessDelegationMode selectAccessDelegationMode( + Set delegationModes) { + return delegationModes.contains(VENDED_CREDENTIALS) + ? AccessDelegationMode.VENDED_CREDENTIALS + : delegationModes.contains(REMOTE_SIGNING) + ? AccessDelegationMode.REMOTE_SIGNING + : AccessDelegationMode.UNKNOWN; + } + private void validateRemoteTableLocations( TableIdentifier tableIdentifier, Set tableLocations, @@ -1239,12 +1283,7 @@ private EnumSet getUpdateTableAuthorizableOperatio } } - private void checkAllowExternalCatalogCredentialVending( - EnumSet delegationModes) { - - if (delegationModes.isEmpty()) { - return; - } + private void checkAllowExternalCatalogCredentialVending() { CatalogEntity catalogEntity = getResolvedCatalogEntity(); LOGGER.info("Catalog type: {}", catalogEntity.getCatalogType()); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java index d336040273..ad81a20871 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java @@ -22,16 +22,26 @@ import jakarta.annotation.Nonnull; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; +import java.net.URI; import java.util.Optional; import java.util.Set; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.rest.PolarisResourcePaths; import org.apache.polaris.core.storage.AccessConfig; import org.apache.polaris.core.storage.PolarisStorageActions; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; +import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,13 +59,22 @@ public class AccessConfigProvider { private final StorageCredentialCache storageCredentialCache; private final MetaStoreManagerFactory metaStoreManagerFactory; + private final PolarisStorageIntegrationProvider storageIntegrationProvider; + private final CatalogPrefixParser prefixParser; + private final UriInfo uriInfo; @Inject public AccessConfigProvider( StorageCredentialCache storageCredentialCache, - MetaStoreManagerFactory metaStoreManagerFactory) { + MetaStoreManagerFactory metaStoreManagerFactory, + PolarisStorageIntegrationProvider storageIntegrationProvider, + CatalogPrefixParser prefixParser, + UriInfo uriInfo) { this.storageCredentialCache = storageCredentialCache; this.metaStoreManagerFactory = metaStoreManagerFactory; + this.storageIntegrationProvider = storageIntegrationProvider; + this.prefixParser = prefixParser; + this.uriInfo = uriInfo; } /** @@ -71,7 +90,7 @@ public AccessConfigProvider( * @return {@link AccessConfig} with scoped credentials and metadata; empty if no storage config * found */ - public AccessConfig getAccessConfig( + public AccessConfig getAccessConfigForCredentialsVending( @Nonnull CallContext callContext, @Nonnull TableIdentifier tableIdentifier, @Nonnull Set tableLocations, @@ -89,9 +108,9 @@ public AccessConfig getAccessConfig( .atWarn() .addKeyValue("tableIdentifier", tableIdentifier) .log("Table entity has no storage configuration in its hierarchy"); - return AccessConfig.builder().supportsCredentialVending(false).build(); + return AccessConfig.EMPTY; } - return FileIOUtil.refreshAccessConfig( + return FileIOUtil.refreshAccessConfigWithCredentialSubscoping( callContext, storageCredentialCache, metaStoreManagerFactory.getOrCreateMetaStoreManager(callContext.getRealmContext()), @@ -101,4 +120,58 @@ public AccessConfig getAccessConfig( storageInfo.get(), refreshCredentialsEndpoint); } + + /** + * Generates a remote signing configuration for accessing table storage at explicit locations. + * + * @param callContext the call context containing realm, principal, and security context + * @param catalogName the name of the catalog + * @param tableIdentifier the table identifier, used for logging and refresh endpoint construction + * @param resolvedPath the entity hierarchy to search for storage configuration + * @return {@link AccessConfig} with scoped credentials and metadata; empty if no storage config + * found + */ + public AccessConfig getAccessConfigForRemoteSigning( + @Nonnull CallContext callContext, + @Nonnull String catalogName, + @Nonnull TableIdentifier tableIdentifier, + @Nonnull PolarisResolvedPathWrapper resolvedPath) { + LOGGER + .atDebug() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Fetching remote signing config for table"); + + Optional storageInfo = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + Optional configurationInfo = + storageInfo + .map(PolarisEntity::getInternalPropertiesAsMap) + .map(info -> info.get(PolarisEntityConstants.getStorageConfigInfoPropertyName())) + .map(PolarisStorageConfigurationInfo::deserialize); + + if (configurationInfo.isEmpty()) { + LOGGER + .atWarn() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Table entity has no storage configuration in its hierarchy"); + return AccessConfig.EMPTY; + } + + PolarisStorageIntegration storageIntegration = + storageIntegrationProvider.getStorageIntegrationForConfig(configurationInfo.get()); + + if (!(storageIntegration + instanceof AwsCredentialsStorageIntegration awsCredentialsStorageIntegration)) { + LOGGER + .atWarn() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Table entity storage integration is not an AWS credentials storage integration"); + return AccessConfig.EMPTY; + } + + String prefix = prefixParser.catalogNameToPrefix(callContext.getRealmContext(), catalogName); + URI signerUri = uriInfo.getBaseUri().resolve("api/"); + String signerEndpoint = new PolarisResourcePaths(prefix).s3RemoteSigning(tableIdentifier); + + return awsCredentialsStorageIntegration.getRemoteSigningAccessConfig(signerUri, signerEndpoint); + } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java index f4a6320d67..127e123e75 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/FileIOUtil.java @@ -63,7 +63,8 @@ public static Optional findStorageInfoFromHierarchy( } /** - * Refreshes or generates subscoped creds for accessing table storage based on the params. + * Refreshes or generates subscoped creds for accessing table storage based on the params. Returns + * empty if subscoped credentials are disabled. * *

Use cases: * @@ -74,7 +75,7 @@ public static Optional findStorageInfoFromHierarchy( * and read/write metadata JSON files. * */ - public static AccessConfig refreshAccessConfig( + public static AccessConfig refreshAccessConfigWithCredentialSubscoping( CallContext callContext, StorageCredentialCache storageCredentialCache, PolarisCredentialVendor credentialVendor, @@ -93,7 +94,7 @@ public static AccessConfig refreshAccessConfig( .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) .log("Skipping generation of subscoped creds for table"); - return AccessConfig.builder().build(); + return AccessConfig.EMPTY; } boolean allowList = diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 23ec20abc3..46f64c6489 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -53,13 +53,12 @@ public class PolarisStorageIntegrationProviderImpl implements PolarisStorageInte private final Optional stsCredentials; private final Supplier gcpCredsProvider; - @SuppressWarnings("CdiInjectionPointsInspection") @Inject public PolarisStorageIntegrationProviderImpl( StorageConfiguration storageConfiguration, StsClientProvider stsClientProvider, Clock clock) { this( stsClientProvider, - Optional.ofNullable(storageConfiguration.stsCredentials()), + storageConfiguration.awsSystemCredentials(), storageConfiguration.gcpCredentialsSupplier(clock)); } @@ -115,7 +114,7 @@ public AccessConfig getSubscopedCreds( @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations, Optional refreshCredentialsEndpoint) { - return AccessConfig.builder().supportsCredentialVending(false).build(); + return AccessConfig.EMPTY; } @Override diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java index f92c45416a..a76fad8f77 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java @@ -33,7 +33,6 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.StsClientBuilder; @@ -72,28 +71,32 @@ public interface StorageConfiguration extends S3AccessConfig { Optional gcpAccessTokenLifespan(); default Supplier stsClientSupplier() { - return stsClientSupplier(true); - } - - default Supplier stsClientSupplier(boolean withCredentials) { return Suppliers.memoize( () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - if (withCredentials) { - stsClientBuilder.credentialsProvider(stsCredentials()); - } - return stsClientBuilder.build(); + StsClientBuilder builder = StsClient.builder(); + awsSystemCredentials().ifPresent(builder::credentialsProvider); + return builder.build(); }); } - default AwsCredentialsProvider stsCredentials() { + /** + * Returns an {@link AwsCredentialsProvider} that provides system-wide AWS credentials. If both + * access key and secret key are present, a static credentials provider is returned. Otherwise, an + * empty optional is returned, implying that the default credentials provider chain should be + * used. + * + *

The returned provider is not meant to be vended directly to clients, but rather used with + * STS, unless credential subscoping is disabled. + */ + default Optional awsSystemCredentials() { if (awsAccessKey().isPresent() && awsSecretKey().isPresent()) { LoggerFactory.getLogger(StorageConfiguration.class) .warn("Using hard-coded AWS credentials - this is not recommended for production"); - return StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get())); + return Optional.of( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get()))); } else { - return DefaultCredentialsProvider.builder().build(); + return Optional.empty(); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java new file mode 100644 index 0000000000..eec278e7a4 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import java.util.function.Function; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.service.catalog.CatalogPrefixParser; +import org.apache.polaris.service.catalog.common.CatalogAdapter; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.api.IcebergRestS3SignerApiService; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** A bridge between {@link IcebergRestS3SignerApiService} and {@link CatalogAdapter}. */ +@RequestScoped +public class S3RemoteSigningCatalogAdapter + implements IcebergRestS3SignerApiService, CatalogAdapter { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3RemoteSigningCatalogAdapter.class); + + @Inject PolarisDiagnostics diagnostics; + @Inject CallContext callContext; + @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisAuthorizer polarisAuthorizer; + @Inject CatalogPrefixParser prefixParser; + @Inject S3RequestSigner s3RequestSigner; + @Inject CallContextCatalogFactory callContextCatalogFactory; + @Inject PolarisPrincipal polarisPrincipal; + + /** + * Execute operations on a catalog wrapper and ensure we close the BaseCatalog afterward. This + * will typically ensure the underlying FileIO is closed. + */ + private Response withCatalog( + SecurityContext securityContext, + String prefix, + Function action) { + validatePrincipal(securityContext); + String catalogName = prefixParser.prefixToCatalogName(callContext.getRealmContext(), prefix); + try (S3RemoteSigningCatalogHandler handler = + new S3RemoteSigningCatalogHandler( + diagnostics, + callContext, + resolutionManifestFactory, + callContextCatalogFactory, + polarisPrincipal, + catalogName, + polarisAuthorizer, + s3RequestSigner)) { + return action.apply(handler); + } catch (RuntimeException e) { + LOGGER.debug("RuntimeException while operating on catalog. Propagating to caller.", e); + throw e; + } catch (Exception e) { + LOGGER.error("Error while operating on catalog", e); + throw new RuntimeException(e); + } + } + + @Override + public Response signS3Request( + String prefix, + String namespace, + String table, + PolarisS3SignRequest s3SignRequest, + RealmContext realmContext, + SecurityContext securityContext) { + Namespace ns = decodeNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); + return withCatalog( + securityContext, + prefix, + catalog -> { + S3SignResponse response = catalog.signS3Request(s3SignRequest, tableIdentifier); + return Response.status(Response.Status.OK).entity(response).build(); + }); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java new file mode 100644 index 0000000000..26d668826a --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Optional; +import java.util.Set; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageActions; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.StorageUtil; +import org.apache.polaris.service.catalog.common.CatalogHandler; +import org.apache.polaris.service.catalog.common.CatalogUtils; +import org.apache.polaris.service.catalog.io.FileIOUtil; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3RemoteSigningCatalogHandler extends CatalogHandler implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3RemoteSigningCatalogHandler.class); + + private final CallContextCatalogFactory catalogFactory; + private final S3RequestSigner s3RequestSigner; + + private CatalogEntity catalogEntity; + private Catalog baseCatalog; + + public S3RemoteSigningCatalogHandler( + PolarisDiagnostics diagnostics, + CallContext callContext, + ResolutionManifestFactory resolutionManifestFactory, + CallContextCatalogFactory catalogFactory, + PolarisPrincipal polarisPrincipal, + String catalogName, + PolarisAuthorizer authorizer, + S3RequestSigner s3RequestSigner) { + super( + diagnostics, + callContext, + resolutionManifestFactory, + polarisPrincipal, + catalogName, + authorizer, + // external catalogs are not supported for S3 remote signing + null, + null); + this.catalogFactory = catalogFactory; + this.s3RequestSigner = s3RequestSigner; + } + + @Override + protected void initializeCatalog() { + catalogEntity = + CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + if (catalogEntity.isExternal()) { + throw new ForbiddenException("Cannot use S3 remote signing with federated catalogs."); + } + baseCatalog = + catalogFactory.createCallContextCatalog(callContext, polarisPrincipal, resolutionManifest); + } + + public PolarisS3SignResponse signS3Request( + PolarisS3SignRequest s3SignRequest, TableIdentifier tableIdentifier) { + + LOGGER.debug("Requesting s3 signing for {}: {}", tableIdentifier, s3SignRequest); + + PolarisAuthorizableOperation authzOp = + s3SignRequest.write() + ? PolarisAuthorizableOperation.SIGN_S3_WRITE_REQUEST + : PolarisAuthorizableOperation.SIGN_S3_READ_REQUEST; + + authorizeRemoteSigningOrThrow(EnumSet.of(authzOp), tableIdentifier); + + // Must be done after the authorization check, as the auth check creates the catalog entity + throwIfRemoteSigningNotEnabled(callContext.getRealmConfig(), catalogEntity); + + var result = + InMemoryStorageIntegration.validateAllowedLocations( + callContext.getRealmConfig(), + getAllowedLocations(tableIdentifier), + getStorageActions(s3SignRequest), + getTargetLocations(s3SignRequest)); + + if (result.values().stream().anyMatch(r -> r.values().stream().anyMatch(v -> !v.isSuccess()))) { + throw new ForbiddenException("Requested S3 location is not allowed."); + } + + PolarisS3SignResponse s3SignResponse = s3RequestSigner.signRequest(s3SignRequest); + LOGGER.debug("S3 signing response: {}", s3SignResponse); + + return s3SignResponse; + } + + private Collection getAllowedLocations(TableIdentifier tableIdentifier) { + + if (baseCatalog.tableExists(tableIdentifier)) { + + // If the table exists, get allowed locations from the table metadata + Table table = baseCatalog.loadTable(tableIdentifier); + if (table instanceof BaseTable baseTable) { + return StorageUtil.getLocationsUsedByTable(baseTable.operations().current()); + } + + throw new ForbiddenException("No storage configuration found for table."); + + } else { + + // If the table or view doesn't exist, the engine might be writing the manifests before the + // table creation is committed. In this case, we still need to check allowed locations from + // the parent entities. + + PolarisResolvedPathWrapper resolvedPath = + CatalogUtils.findResolvedStorageEntity(resolutionManifest, tableIdentifier); + + Optional storageInfo = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + + var configurationInfo = + storageInfo + .map(PolarisEntity::getInternalPropertiesAsMap) + .map(info -> info.get(PolarisEntityConstants.getStorageConfigInfoPropertyName())) + .map(PolarisStorageConfigurationInfo::deserialize); + + if (configurationInfo.isEmpty()) { + throw new ForbiddenException("No storage configuration found for table."); + } + + return configurationInfo.get().getAllowedLocations(); + } + } + + private Set getStorageActions(PolarisS3SignRequest s3SignRequest) { + // TODO M2: better handling of DELETE and LIST + return s3SignRequest.write() + ? Set.of(PolarisStorageActions.WRITE) + : Set.of(PolarisStorageActions.READ); + } + + private Set getTargetLocations(PolarisS3SignRequest s3SignRequest) { + // TODO M2: map http URI to s3 URI + return Set.of(); + } + + public static void throwIfRemoteSigningNotEnabled( + RealmConfig realmConfig, CatalogEntity catalogEntity) { + if (catalogEntity.isExternal()) { + throw new ForbiddenException("Remote signing is not enabled for external catalogs."); + } + boolean remoteSigningEnabled = + realmConfig.getConfig(FeatureConfiguration.REMOTE_SIGNING_ENABLED, catalogEntity); + if (!remoteSigningEnabled) { + throw new ForbiddenException( + "Remote signing is not enabled for this catalog. To enable this feature, set the Polaris configuration %s " + + "or the catalog configuration %s.", + FeatureConfiguration.REMOTE_SIGNING_ENABLED.key(), + FeatureConfiguration.REMOTE_SIGNING_ENABLED.catalogConfig()); + } + } + + @Override + public void close() {} +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java new file mode 100644 index 0000000000..63175338be --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; + +/** Interface for signing S3 requests. */ +public interface S3RequestSigner { + + /** Signs an S3 request. */ + PolarisS3SignResponse signRequest(S3SignRequest signingRequest); +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java new file mode 100644 index 0000000000..c2ae7db79a --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.net.URI; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignResponse; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; +import org.apache.polaris.service.storage.StorageConfiguration; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner; +import software.amazon.awssdk.http.auth.spi.signer.SignRequest; +import software.amazon.awssdk.http.auth.spi.signer.SignedRequest; +import software.amazon.awssdk.services.s3.S3Client; + +@ApplicationScoped +class S3RequestSignerImpl implements S3RequestSigner { + + private final AwsV4HttpSigner signer = AwsV4HttpSigner.create(); + + @Inject StorageConfiguration storageConfiguration; + + @Override + public PolarisS3SignResponse signRequest(S3SignRequest signingRequest) { + + URI uri = signingRequest.uri(); + SdkHttpMethod method = SdkHttpMethod.valueOf(signingRequest.method()); + + SdkHttpFullRequest.Builder requestToSign = + SdkHttpFullRequest.builder() + .uri(uri) + .protocol(uri.getScheme()) + .method(method) + .headers(signingRequest.headers()); + + AwsCredentials credentials = + storageConfiguration + .awsSystemCredentials() + .orElseGet(() -> DefaultCredentialsProvider.builder().build()) + .resolveCredentials(); + + SignRequest.Builder signRequest = + SignRequest.builder(credentials) + .request(requestToSign.build()) + .putProperty(AwsV4HttpSigner.REGION_NAME, signingRequest.region()) + .putProperty(AwsV4HttpSigner.SERVICE_SIGNING_NAME, S3Client.SERVICE_NAME) + .putProperty(AwsV4HttpSigner.DOUBLE_URL_ENCODE, false) + .putProperty(AwsV4HttpSigner.NORMALIZE_PATH, false) + .putProperty(AwsV4HttpSigner.CHUNK_ENCODING_ENABLED, false) + .putProperty(AwsV4HttpSigner.PAYLOAD_SIGNING_ENABLED, false); + + String body = signingRequest.body(); + if (body != null) { + signRequest.payload(ContentStreamProvider.fromUtf8String(body)); + } + + SignedRequest signed = signer.sign(signRequest.build()); + SdkHttpRequest signedRequest = signed.request(); + + return ImmutablePolarisS3SignResponse.builder() + .uri(signedRequest.getUri()) + .headers(signedRequest.headers()) + .build(); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index 720f204fc2..7feebfc10a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -63,7 +63,7 @@ public FileIO apply(TaskEntity task, TableIdentifier identifier, CallContext cal PolarisResolvedPathWrapper resolvedPath = new PolarisResolvedPathWrapper(List.of(resolvedTaskEntity)); AccessConfig accessConfig = - accessConfigProvider.getAccessConfig( + accessConfigProvider.getAccessConfigForCredentialsVending( callContext, identifier, locations, storageActions, Optional.empty(), resolvedPath); String ioImpl = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 4b1799d8cf..a200ca4715 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -81,6 +81,7 @@ import org.apache.polaris.core.policy.PredefinedPolicyTypes; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; @@ -91,7 +92,6 @@ import org.apache.polaris.service.catalog.io.FileIOFactory; import org.apache.polaris.service.catalog.policy.PolicyCatalog; import org.apache.polaris.service.config.ReservedProperties; -import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.context.catalog.PolarisCallContextCatalogFactory; import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; @@ -127,6 +127,7 @@ public Map getConfigOverrides() { .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") .build(); } } @@ -192,7 +193,6 @@ public Map getConfigOverrides() { @Inject protected MetaStoreManagerFactory managerFactory; @Inject protected ResolutionManifestFactory resolutionManifestFactory; - @Inject protected CallContextCatalogFactory callContextCatalogFactory; @Inject protected UserSecretsManagerFactory userSecretsManagerFactory; @Inject protected ServiceIdentityProvider serviceIdentityProvider; @Inject protected PolarisCredentialManager credentialManager; @@ -204,6 +204,7 @@ public Map getConfigOverrides() { @Inject protected StorageCredentialCache storageCredentialCache; @Inject protected ResolverFactory resolverFactory; @Inject protected AccessConfigProvider accessConfigProvider; + @Inject protected PolarisStorageIntegrationProvider storageIntegrationProvider; protected IcebergCatalog baseCatalog; protected PolarisGenericTableCatalog genericTableCatalog; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java index f4dceffe62..38057e8d42 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Method; import java.util.List; @@ -64,6 +65,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.AccessConfigProvider; @@ -106,6 +108,7 @@ public abstract class AbstractPolarisGenericTableCatalogTest { @Inject PolarisDiagnostics diagServices; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject CatalogPrefixParser prefixParser; private PolarisGenericTableCatalog genericTableCatalog; private IcebergCatalog icebergCatalog; @@ -157,7 +160,12 @@ public void before(TestInfo testInfo) { configurationStore); realmConfig = polarisContext.getRealmConfig(); accessConfigProvider = - new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory); + new AccessConfigProvider( + storageCredentialCache, + metaStoreManagerFactory, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java index fa05af7efd..5c716891ee 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java @@ -38,6 +38,7 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.reflect.Method; @@ -134,6 +135,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.AccessConfigProvider; @@ -236,6 +238,7 @@ public Map getConfigOverrides() { @Inject ServiceIdentityProvider serviceIdentityProvider; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject CatalogPrefixParser prefixParser; private IcebergCatalog catalog; private String realmName; @@ -291,7 +294,12 @@ public void before(TestInfo testInfo) { configurationStore); realmConfig = polarisContext.getRealmConfig(); accessConfigProvider = - new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory); + new AccessConfigProvider( + storageCredentialCache, + metaStoreManagerFactory, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); EntityCache entityCache = createEntityCache(diagServices, realmConfig, metaStoreManager); resolverFactory = (principal, referenceCatalogName) -> diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java index 2406108abb..cbd7d97183 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -53,8 +54,10 @@ import org.apache.polaris.core.persistence.resolver.ResolverFactory; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.AccessConfigProvider; @@ -113,6 +116,8 @@ public Map getConfigOverrides() { @Inject PolarisEventListener polarisEventListener; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject CatalogPrefixParser prefixParser; private IcebergCatalog catalog; @@ -163,7 +168,12 @@ public void before(TestInfo testInfo) { configurationStore); realmConfig = polarisContext.getRealmConfig(); accessConfigProvider = - new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory); + new AccessConfigProvider( + storageCredentialCache, + metaStoreManagerFactory, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); PolarisPrincipal authenticatedRoot = PolarisPrincipal.of(rootPrincipal, Set.of()); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java index 924e376e4b..eb9ac399d9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java @@ -24,6 +24,7 @@ import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.time.Instant; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,6 +70,7 @@ import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.context.catalog.PolarisCallContextCatalogFactory; @@ -642,7 +644,10 @@ public void testCreateTableDirectWithWriteDelegationAllSufficientPrivileges() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableDirectWithWriteDelegation( - NS2, createDirectWithWriteDelegationRequest, Optional.empty()); + NS2, + createDirectWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }, () -> { newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithPurge(newtable); @@ -673,7 +678,10 @@ public void testCreateTableDirectWithWriteDelegationInsufficientPermissions() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableDirectWithWriteDelegation( - NS2, createDirectWithWriteDelegationRequest, Optional.empty()); + NS2, + createDirectWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }); } @@ -747,7 +755,10 @@ public void testCreateTableStagedWithWriteDelegationAllSufficientPrivileges() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableStagedWithWriteDelegation( - NS2, createStagedWithWriteDelegationRequest, Optional.empty()); + NS2, + createStagedWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }, // createTableStagedWithWriteDelegation doesn't actually commit any metadata null, @@ -777,7 +788,10 @@ public void testCreateTableStagedWithWriteDelegationInsufficientPermissions() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableStagedWithWriteDelegation( - NS2, createStagedWithWriteDelegationRequest, Optional.empty()); + NS2, + createStagedWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }); } @@ -921,7 +935,13 @@ public void testLoadTableWithReadAccessDelegationSufficientPrivileges() { PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty()), + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -937,7 +957,13 @@ public void testLoadTableWithReadAccessDelegationInsufficientPermissions() { PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_LIST, PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty())); + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test @@ -950,7 +976,13 @@ public void testLoadTableWithWriteAccessDelegationSufficientPrivileges() { PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty()), + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -966,7 +998,13 @@ public void testLoadTableWithWriteAccessDelegationInsufficientPermissions() { PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_LIST, PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty())); + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test @@ -979,7 +1017,11 @@ public void testLoadTableWithReadAccessDelegationIfStaleSufficientPrivileges() { () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty()), + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -998,7 +1040,51 @@ public void testLoadTableWithReadAccessDelegationIfStaleInsufficientPermissions( () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty())); + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); + } + + @Test + public void testLoadTableWithReadRemoteSigningIfStaleSufficientPrivileges() { + doTestSufficientPrivileges( + List.of( + PolarisPrivilege.TABLE_READ_DATA, + PolarisPrivilege.TABLE_WRITE_DATA, + PolarisPrivilege.CATALOG_MANAGE_CONTENT), + () -> + newWrapper() + .loadTableWithAccessDelegationIfStale( + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.REMOTE_SIGNING), + Optional.empty()), + null /* cleanupAction */); + } + + @Test + public void testLoadTableWithReadRemoteSigningIfStaleInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of( + PolarisPrivilege.NAMESPACE_FULL_METADATA, + PolarisPrivilege.VIEW_FULL_METADATA, + PolarisPrivilege.TABLE_FULL_METADATA, + PolarisPrivilege.TABLE_READ_PROPERTIES, + PolarisPrivilege.TABLE_WRITE_PROPERTIES, + PolarisPrivilege.TABLE_CREATE, + PolarisPrivilege.TABLE_LIST, + PolarisPrivilege.TABLE_DROP), + () -> + newWrapper() + .loadTableWithAccessDelegationIfStale( + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.REMOTE_SIGNING), + Optional.empty())); } @Test @@ -1014,7 +1100,11 @@ public void testLoadTableWithWriteAccessDelegationIfStaleSufficientPrivileges() () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty()), + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -1033,7 +1123,11 @@ public void testLoadTableWithWriteAccessDelegationIfStaleInsufficientPermissions () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty())); + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java index 69b9cde7b6..7c6444cd12 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Method; import java.util.Arrays; @@ -76,6 +77,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.AccessConfigProvider; @@ -132,6 +134,7 @@ public abstract class AbstractPolicyCatalogTest { @Inject PolarisDiagnostics diagServices; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject CatalogPrefixParser prefixParser; private PolicyCatalog policyCatalog; private IcebergCatalog icebergCatalog; @@ -178,7 +181,12 @@ public void before(TestInfo testInfo) { configurationStore); realmConfig = polarisContext.getRealmConfig(); accessConfigProvider = - new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory); + new AccessConfigProvider( + storageCredentialCache, + metaStoreManagerFactory, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java new file mode 100644 index 0000000000..57ab3baac3 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX; +import static org.apache.iceberg.aws.AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.ACCESS_KEY_ID; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.ENDPOINT; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.SECRET_ACCESS_KEY; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_KEY_ID; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY; +import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.PositionOutputStream; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.types.Types; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.catalog.AccessDelegationMode; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * These tests complement {@link PolarisRestCatalogMinIOIT} to validate client-side access to MinIO + * storage via {@code FileIO} instances configured from catalog's {@code loadTable} responses with + * some S3-specific options. + */ +@QuarkusTest +@TestProfile(RestCatalogMinIOSpecialTest.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class RestCatalogMinIOSpecialTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + private static String adminToken; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.IntegerType.get(), "doc"), + optional(2, "data", Types.StringType.get())); + + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + private static URI storageBase; + private static String endpoint; + private static S3Client s3Client; + + private CatalogApi catalogApi; + private String principalRoleName; + private PrincipalWithCredentials principalCredentials; + private String catalogName; + + @BeforeAll + static void setup( + PolarisApiEndpoints apiEndpoints, + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess, + ClientCredentials credentials) { + s3Client = minioAccess.s3Client(); + endpoints = apiEndpoints; + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @AfterAll + static void close() throws Exception { + client.close(); + } + + @BeforeEach + public void before(TestInfo testInfo) { + String principalName = client.newEntityName("test-user"); + principalRoleName = client.newEntityName("test-admin"); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + + String principalToken = client.obtainToken(principalCredentials); + catalogApi = client.catalogApi(principalToken); + + catalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName()); + } + + private RESTCatalog createCatalog( + Optional endpoint, + Optional stsEndpoint, + boolean pathStyleAccess, + Optional delegationMode, + boolean stsEnabled) { + return createCatalog( + endpoint, stsEndpoint, pathStyleAccess, Optional.empty(), delegationMode, stsEnabled); + } + + private RESTCatalog createCatalog( + Optional endpoint, + Optional stsEndpoint, + boolean pathStyleAccess, + Optional endpointInternal, + Optional delegationMode, + boolean stsEnabled) { + AwsStorageConfigInfo.Builder storageConfig = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setPathStyleAccess(pathStyleAccess) + .setStsUnavailable(!stsEnabled) + .setAllowedLocations(List.of(storageBase.toString())); + + endpoint.ifPresent(storageConfig::setEndpoint); + stsEndpoint.ifPresent(storageConfig::setStsEndpoint); + endpointInternal.ifPresent(storageConfig::setEndpointInternal); + + CatalogProperties.Builder catalogProps = + CatalogProperties.builder(storageBase.toASCIIString() + "/" + catalogName); + if (!stsEnabled) { + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY); + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setStorageConfigInfo(storageConfig.build()) + .setProperties(catalogProps.build()) + .build(); + + managementApi.createCatalog(principalRoleName, catalog); + + String authToken = client.obtainToken(principalCredentials); + RESTCatalog restCatalog = new RESTCatalog(); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put( + org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) + .put(OAuth2Properties.TOKEN, authToken) + .put("warehouse", catalogName) + .putAll(endpoints.extraHeaders("header.")); + + delegationMode.ifPresent( + dm -> propertiesBuilder.put("header.X-Iceberg-Access-Delegation", dm.protocolValue())); + + if (delegationMode.isEmpty()) { + // Use local credentials on the client side + propertiesBuilder.put("s3.access-key-id", MINIO_ACCESS_KEY); + propertiesBuilder.put("s3.secret-access-key", MINIO_SECRET_KEY); + } + + restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + return restCatalog; + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminToken); + } + + @ParameterizedTest + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + public void testCreateTable(boolean pathStyle, boolean stsEnabled) throws IOException { + LoadTableResponse response = doTestCreateTable(pathStyle, Optional.empty(), stsEnabled); + assertThat(response.config()).doesNotContainKey(SECRET_ACCESS_KEY); + assertThat(response.config()).doesNotContainKey(ACCESS_KEY_ID); + assertThat(response.config()).doesNotContainKey(REFRESH_CREDENTIALS_ENDPOINT); + assertThat(response.credentials()).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testCreateTableVendedCredentials(boolean pathStyle) throws IOException { + LoadTableResponse response = + doTestCreateTable(pathStyle, Optional.of(VENDED_CREDENTIALS), true); + assertThat(response.config()) + .containsEntry( + REFRESH_CREDENTIALS_ENDPOINT, + "v1/" + catalogName + "/namespaces/test-ns/tables/t1/credentials"); + assertThat(response.credentials()).hasSize(1); + } + + private LoadTableResponse doTestCreateTable( + boolean pathStyle, Optional dm, boolean stsEnabled) throws IOException { + try (RESTCatalog restCatalog = + createCatalog(Optional.of(endpoint), Optional.empty(), pathStyle, dm, stsEnabled)) { + LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog, dm); + if (pathStyle) { + assertThat(loadTableResponse.config()) + .containsEntry("s3.path-style-access", Boolean.TRUE.toString()); + } + return loadTableResponse; + } + } + + @Test + public void testInternalEndpoints() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of("http://s3.example.com"), + Optional.of(endpoint), + false, + Optional.of(endpoint), + Optional.empty(), + true)) { + StorageConfigInfo storageConfig = + managementApi.getCatalog(catalogName).getStorageConfigInfo(); + assertThat((AwsStorageConfigInfo) storageConfig) + .extracting( + AwsStorageConfigInfo::getEndpoint, + AwsStorageConfigInfo::getStsEndpoint, + AwsStorageConfigInfo::getEndpointInternal, + AwsStorageConfigInfo::getPathStyleAccess) + .containsExactly("http://s3.example.com", endpoint, endpoint, false); + LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog, Optional.empty()); + assertThat(loadTableResponse.config()).containsEntry(ENDPOINT, "http://s3.example.com"); + } + } + + @Test + public void testCreateTableFailureWithCredentialVendingWithoutSts() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of("http://sts.example.com"), // not called + false, + Optional.of(VENDED_CREDENTIALS), + false)) { + StorageConfigInfo storageConfig = + managementApi.getCatalog(catalogName).getStorageConfigInfo(); + assertThat((AwsStorageConfigInfo) storageConfig) + .extracting( + AwsStorageConfigInfo::getEndpoint, + AwsStorageConfigInfo::getStsEndpoint, + AwsStorageConfigInfo::getEndpointInternal, + AwsStorageConfigInfo::getPathStyleAccess, + AwsStorageConfigInfo::getStsUnavailable) + .containsExactly(endpoint, "http://sts.example.com", null, false, true); + + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t2"); + // Credential vending is not supported without STS + assertThatThrownBy(() -> restCatalog.createTable(id, SCHEMA)) + .hasMessageContaining("but no credentials are available") + .hasMessageContaining(id.toString()); + } + } + + @Test + public void testLoadTableFailureWithCredentialVendingWithoutSts() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of("http://sts.example.com"), // not called + false, + Optional.empty(), + false)) { + + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t3"); + restCatalog.createTable(id, SCHEMA); + + // Credential vending is not supported without STS + assertThatThrownBy( + () -> + catalogApi.loadTable( + catalogName, + id, + "ALL", + Map.of("X-Iceberg-Access-Delegation", VENDED_CREDENTIALS.protocolValue()))) + .hasMessageContaining("but no credentials are available") + .hasMessageContaining(id.toString()); + } + } + + public LoadTableResponse doTestCreateTable( + RESTCatalog restCatalog, Optional dm) { + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t1"); + Table table = restCatalog.createTable(id, SCHEMA); + assertThat(table).isNotNull(); + assertThat(restCatalog.tableExists(id)).isTrue(); + + TableOperations ops = ((HasTableOperations) table).operations(); + URI location = URI.create(ops.current().metadataFileLocation()); + + GetObjectResponse response = + s3Client + .getObject( + GetObjectRequest.builder() + .bucket(location.getAuthority()) + .key(location.getPath().substring(1)) // drop leading slash + .build()) + .response(); + assertThat(response.contentLength()).isGreaterThan(0); + + LoadTableResponse loadTableResponse = + catalogApi.loadTable( + catalogName, + id, + "ALL", + dm.map(v -> Map.of("X-Iceberg-Access-Delegation", v.protocolValue())).orElse(Map.of())); + + assertThat(loadTableResponse.config()).containsKey(ENDPOINT); + + restCatalog.dropTable(id); + assertThat(restCatalog.tableExists(id)).isFalse(); + return loadTableResponse; + } + + @ParameterizedTest + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + @CsvSource("true, true, VENDED_CREDENTIALS") + @CsvSource("false, true, VENDED_CREDENTIALS") + public void testAppendFiles( + boolean pathStyle, boolean stsEnabled, AccessDelegationMode delegationMode) + throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of(endpoint), + pathStyle, + Optional.ofNullable(delegationMode), + stsEnabled)) { + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t1"); + Table table = restCatalog.createTable(id, SCHEMA); + assertThat(table).isNotNull(); + + @SuppressWarnings("resource") + FileIO io = table.io(); + + URI loc = + URI.create( + table + .locationProvider() + .newDataLocation( + String.format( + "test-file-%s-%s-%s.txt", pathStyle, delegationMode, stsEnabled))); + OutputFile f1 = io.newOutputFile(loc.toString()); + try (PositionOutputStream os = f1.create()) { + os.write("Hello World".getBytes(UTF_8)); + } + + DataFile df = + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(f1.location()) + .withFormat(FileFormat.PARQUET) // bogus value + .withFileSizeInBytes(4) + .withRecordCount(1) + .build(); + + table.newAppend().appendFile(df).commit(); + + try (InputStream is = + s3Client.getObject( + GetObjectRequest.builder() + .bucket(loc.getAuthority()) + .key(loc.getPath().substring(1)) // drop leading slash + .build())) { + assertThat(new String(is.readAllBytes(), UTF_8)).isEqualTo("Hello World"); + } + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java new file mode 100644 index 0000000000..a3968a8b45 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.service.it.test.PolarisS3RemoteSigningIntegrationTest; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusTest +@TestProfile(S3RemoteSigningMinIOIntegrationTest.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class S3RemoteSigningMinIOIntegrationTest extends PolarisS3RemoteSigningIntegrationTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static URI storageBase; + private static String endpoint; + + @BeforeAll + static void setup( + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess) { + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @Override + protected List allowedLocations() { + return List.of(storageBase.toString()); + } + + @Override + protected Optional endpoint() { + return Optional.of(endpoint); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + return super.clientFileIOProperties() + // Grant direct access to the MinIO bucket; this FileIO instance does not + // use access delegation. + .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) + .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java index 916b1912a1..91c9db8643 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java @@ -192,7 +192,7 @@ public void testSingletonStsClientWithStaticCredentials() { staticMock.when(StsClient::builder).thenReturn(mockBuilder); StorageConfiguration config = configWithAwsCredentialsAndGcpToken(); - Supplier supplier = config.stsClientSupplier(true); + Supplier supplier = config.stsClientSupplier(); StsClient client1 = supplier.get(); StsClient client2 = supplier.get(); @@ -209,7 +209,7 @@ public void testSingletonStsClientWithStaticCredentials() { @Test public void testStaticStsCredentials() { StorageConfiguration config = configWithAwsCredentialsAndGcpToken(); - AwsCredentialsProvider credentialsProvider = config.stsCredentials(); + AwsCredentialsProvider credentialsProvider = config.awsSystemCredentials().orElseThrow(); assertThat(credentialsProvider).isInstanceOf(StaticCredentialsProvider.class); assertThat(credentialsProvider.resolveCredentials().accessKeyId()).isEqualTo(TEST_ACCESS_KEY); assertThat(credentialsProvider.resolveCredentials().secretAccessKey()) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java new file mode 100644 index 0000000000..c35bdf9cd2 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import static org.mockito.ArgumentMatchers.any; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Set; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignRequest; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignResponse; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +@QuarkusTest +@TestProfile(PolarisAuthzTestBase.Profile.class) +@SuppressWarnings("resource") +public class S3RemoteSigningCatalogHandlerAuthzTest extends PolarisAuthzTestBase { + + private static final ImmutablePolarisS3SignRequest READ_REQUEST = + ImmutablePolarisS3SignRequest.builder() + .method("GET") + .uri(URI.create("https://example-bucket.s3.amazonaws.com/some-object")) + .region("us-west-2") + .build(); + + private static final ImmutablePolarisS3SignRequest WRITE_REQUEST = + ImmutablePolarisS3SignRequest.builder() + .method("PUT") + .uri(URI.create("https://example-bucket.s3.amazonaws.com/some-object")) + .region("us-west-2") + .build(); + + @Test + public void testReadRequestInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of(PolarisPrivilege.TABLE_REMOTE_SIGN, PolarisPrivilege.TABLE_READ_DATA), + () -> newHandler().signS3Request(READ_REQUEST, TABLE_NS1_1)); + } + + @Test + public void testWriteRequestInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of(PolarisPrivilege.TABLE_REMOTE_SIGN, PolarisPrivilege.TABLE_WRITE_DATA), + () -> newHandler().signS3Request(WRITE_REQUEST, TABLE_NS1_1)); + } + + @Test + public void testReadRequestSufficientPermissions() { + doTestSufficientPrivilegeSets( + List.of(Set.of(PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_REMOTE_SIGN)), + () -> newHandler().signS3Request(READ_REQUEST, TABLE_NS1_1), + () -> {}, + PRINCIPAL_NAME, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } + + @Test + public void testWriteRequestSufficientPermissions() { + doTestSufficientPrivilegeSets( + List.of(Set.of(PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.TABLE_REMOTE_SIGN)), + () -> newHandler().signS3Request(WRITE_REQUEST, TABLE_NS1_1), + () -> {}, + PRINCIPAL_NAME, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } + + private S3RemoteSigningCatalogHandler newHandler() { + PolarisPrincipal principal = PolarisPrincipal.of(principalEntity, Set.of()); + S3RequestSigner s3signer = Mockito.mock(S3RequestSigner.class); + Mockito.when(s3signer.signRequest(any())) + .thenReturn(ImmutablePolarisS3SignResponse.builder().uri(URI.create("irrelevant")).build()); + CallContextCatalogFactory callContextCatalogFactory = + Mockito.mock(CallContextCatalogFactory.class); + Mockito.when(callContextCatalogFactory.createCallContextCatalog(any(), any(), any())) + .thenReturn(baseCatalog); + return new S3RemoteSigningCatalogHandler( + diagServices, + callContext, + resolutionManifestFactory, + callContextCatalogFactory, + principal, + CATALOG_NAME, + polarisAuthorizer, + s3signer); + } + + private void doTestInsufficientPrivileges( + List insufficientPrivileges, Runnable action) { + doTestInsufficientPrivileges( + insufficientPrivileges, + PRINCIPAL_NAME, + action, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java new file mode 100644 index 0000000000..1db8a8b4c1 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.aws.s3.signer.ImmutableS3SignRequest; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.polaris.service.storage.StorageConfiguration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +@ExtendWith(MockitoExtension.class) +class S3RequestSignerImplTest { + + @Mock private StorageConfiguration storageConfiguration; + @Mock private AwsCredentialsProvider awsCredentialsProvider; + + @InjectMocks private S3RequestSignerImpl s3RequestSigner; + + private static final String TEST_ACCESS_KEY = "test-access-key"; + private static final String TEST_SECRET_KEY = "test-secret-key"; + private static final String TEST_REGION = "us-west-2"; + private static final String TEST_HOST = "test-bucket.s3.us-west-2.amazonaws.com"; + private static final URI TEST_URI = URI.create("https://" + TEST_HOST + "/test-path"); + + @BeforeEach + void setUp() { + AwsCredentials credentials = AwsBasicCredentials.create(TEST_ACCESS_KEY, TEST_SECRET_KEY); + when(storageConfiguration.awsSystemCredentials()) + .thenReturn(Optional.of(awsCredentialsProvider)); + when(awsCredentialsProvider.resolveCredentials()).thenReturn(credentials); + } + + @Test + void testGET() { + // Given + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testPUT() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("PUT") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testPOST() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("POST") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testDELETE() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("DELETE") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testQueryParameters() { + // Given + URI uriWithQuery = URI.create(TEST_URI + "?prefix=test&max-keys=100"); + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(uriWithQuery) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri().getHost()).isEqualTo(uriWithQuery.getHost()); + assertThat(response.uri().getPath()).isEqualTo(uriWithQuery.getPath()); + assertThat(response.uri().getQuery()).contains("prefix=test"); + assertThat(response.uri().getQuery()).contains("max-keys=100"); + assertHeaders(response); + } + + @Test + void testHeaders() { + // Given + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(TEST_URI) + .headers( + Map.of( + "Host", List.of(TEST_HOST), + "x-amz-content-sha256", List.of("UNSIGNED-PAYLOAD"), + "Content-Type", List.of("application/json"), + "User-Agent", List.of("test-client/1.0"))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + assertThat(response.headers().get("x-amz-content-sha256").getFirst()) + .isEqualTo("UNSIGNED-PAYLOAD"); + assertThat(response.headers().get("Content-Type").getFirst()).isEqualTo("application/json"); + assertThat(response.headers().get("User-Agent").getFirst()).isEqualTo("test-client/1.0"); + } + + private static void assertHeaders(S3SignResponse response) { + assertThat(response.headers()).isNotEmpty(); + assertThat(response.headers()).containsKey("X-Amz-Date"); + assertThat(response.headers()).containsKey("x-amz-content-sha256"); + assertThat(response.headers().get("x-amz-content-sha256").getFirst()) + .isEqualTo("UNSIGNED-PAYLOAD"); // Fixed for S3 + assertThat(response.headers()).containsKey("Authorization"); + assertThat(response.headers().get("Authorization")).hasSize(1); + String authHeader = response.headers().get("Authorization").getFirst(); + assertThat(authHeader).startsWith("AWS4-HMAC-SHA256 Credential=" + TEST_ACCESS_KEY); + assertThat(authHeader).contains("SignedHeaders="); + assertThat(authHeader).contains("Signature="); + } +} diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 030e00b731..c87c11bee7 100644 --- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -27,6 +27,7 @@ import jakarta.annotation.Nullable; import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.UriInfo; import java.security.Principal; import java.time.Clock; import java.time.Instant; @@ -65,6 +66,7 @@ import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.DefaultCatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; @@ -273,8 +275,16 @@ public String getAuthenticationScheme() { PolarisCredentialManager credentialManager = new DefaultPolarisCredentialManager(realmContext, mockCredentialVendors); + CatalogPrefixParser prefixParser = new DefaultCatalogPrefixParser(); + AccessConfigProvider accessConfigProvider = - new AccessConfigProvider(storageCredentialCache, metaStoreManagerFactory); + new AccessConfigProvider( + storageCredentialCache, + metaStoreManagerFactory, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); + FileIOFactory fileIOFactory = fileIOFactorySupplier.get(); TaskExecutor taskExecutor = Mockito.mock(TaskExecutor.class); diff --git a/site/content/in-dev/unreleased/managing-security/access-control.md b/site/content/in-dev/unreleased/managing-security/access-control.md index b8a1b697ca..6ce1cf1757 100644 --- a/site/content/in-dev/unreleased/managing-security/access-control.md +++ b/site/content/in-dev/unreleased/managing-security/access-control.md @@ -102,18 +102,19 @@ To grant the full set of privileges (drop, list, read, write, etc.) on an object ### Table privileges -| Privilege | Description | -| --------- | ----------- | -| TABLE_CREATE | Enables registering a table with the catalog. | -| TABLE_DROP | Enables dropping a table from the catalog. | -| TABLE_LIST | Enables listing any table in the catalog. | -| TABLE_READ_PROPERTIES | Enables reading properties of the table. | -| TABLE_WRITE_PROPERTIES | Enables configuring properties for the table. | -| TABLE_READ_DATA | Enables reading data from the table by receiving short-lived read-only storage credentials from the catalog. | -| TABLE_WRITE_DATA | Enables writing data to the table by receiving short-lived read+write storage credentials from the catalog. | -| TABLE_FULL_METADATA | Grants all table privileges, except TABLE_READ_DATA and TABLE_WRITE_DATA, which need to be granted individually. | -| TABLE_ATTACH_POLICY | Enables attaching policy to a table. | -| TABLE_DETACH_POLICY | Enables detaching policy from a table. | +| Privilege | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------| +| TABLE_CREATE | Enables registering a table with the catalog. | +| TABLE_DROP | Enables dropping a table from the catalog. | +| TABLE_LIST | Enables listing any table in the catalog. | +| TABLE_READ_PROPERTIES | Enables reading properties of the table. | +| TABLE_WRITE_PROPERTIES | Enables configuring properties for the table. | +| TABLE_READ_DATA | Enables reading data from the table by receiving short-lived read-only storage credentials from the catalog. | +| TABLE_WRITE_DATA | Enables writing data to the table by receiving short-lived read+write storage credentials from the catalog. | +| TABLE_FULL_METADATA | Grants all table privileges, except TABLE_READ_DATA and TABLE_WRITE_DATA, which need to be granted individually. | +| TABLE_ATTACH_POLICY | Enables attaching policy to a table. | +| TABLE_DETACH_POLICY | Enables detaching policy from a table. | +| TABLE_REMOTE_SIGN | Enables remote signing for a table. TABLE_READ_DATA and TABLE_WRITE_DATA must also be granted individually. | ### View privileges diff --git a/spec/README.md b/spec/README.md index 8d2c0d0f03..1bc01c21fc 100644 --- a/spec/README.md +++ b/spec/README.md @@ -17,14 +17,40 @@ under the License. --> -# Polaris API Specifications - -Polaris provides two sets of OpenAPI specifications: -- `polaris-management-service.yml` - Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals -- `polaris-catalog-service.yaml` - Defines the specification for the Polaris Catalog API, which encompasses both the Iceberg REST Catalog API - and Polaris-native API. - - `polaris-catalog-apis` - Contains the specification for Polaris-specific Catalog APIs - - `iceberg-rest-catalog-open-api.yaml` - Contains the specification for Iceberg Rest Catalog API +# Apache Polaris API Specifications + +Apache Polaris provides the following OpenAPI specifications: + +- [polaris-management-service.yml](polaris-management-service.yml) - Defines the management APIs for using Apache + Polaris to create and manage Apache Iceberg catalogs and their principals + +- [polaris-catalog-service.yaml](polaris-catalog-service.yaml) - Defines the specification for the Apache Polaris + Catalog API, which encompasses both the Apache Iceberg REST Catalog API and Apache Polaris-native APIs: + + - [iceberg-rest-catalog-open-api.yaml](iceberg-rest-catalog-open-api.yaml) - Contains the specification for + Apache Iceberg Rest Catalog API. + + - [polaris-catalog-apis](polaris-catalog-apis) - This folder contains the specifications for Apache + Polaris-specific Catalog APIs: + + - [generic-tables-api.yaml](polaris-catalog-apis/generic-tables-api.yaml) - Contains the specification for + the Generic Tables API + + - [notifications-api.yaml](polaris-catalog-apis/notifications-api.yaml) - Contains the specification for + the Notifications API + + - [oauth-tokens-api.yaml](polaris-catalog-apis/oauth-tokens-api.yaml) - Contains the specification for the + internal OAuth Token endpoint, extracted from the Apache Iceberg REST Catalog API. + + - [policy-apis.yaml](polaris-catalog-apis/policy-apis.yaml) - Contains the specification for the Policy APIs. + +- [s3-sign](s3-sign) - This folder contains the specifications for S3 remote signing: + + - [iceberg-s3-signer-open-api.yaml](s3-sign/iceberg-s3-signer-open-api.yaml) - Contains the specification of the + original Apache Iceberg S3 Signer API. + + - [polaris-s3-sign-service.yaml](s3-sign/polaris-s3-sign-service.yaml) - Contains the Apache Polaris-specific + S3 Signer API. ## Generated Specification Files The specification files in the generated folder are automatically created using OpenAPI bundling tools such as diff --git a/spec/s3-sign/iceberg-s3-signer-open-api.yaml b/spec/s3-sign/iceberg-s3-signer-open-api.yaml new file mode 100644 index 0000000000..92c17a8e6a --- /dev/null +++ b/spec/s3-sign/iceberg-s3-signer-open-api.yaml @@ -0,0 +1,154 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# CODE_COPIED_TO_POLARIS + +# Apache Iceberg Version: 1.10.0 + +--- +openapi: 3.0.3 +info: + title: Apache Iceberg S3 Signer API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the S3 Signer API. +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be prepended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + basePath: + description: Optional prefix to be appended to all routes + default: "" + +paths: + + /v1/aws/s3/sign: + + post: + tags: + - S3 Signer API + summary: Remotely signs S3 requests + operationId: signS3Request + requestBody: + description: The request containing the headers to be signed + content: + application/json: + schema: + $ref: '#/components/schemas/S3SignRequest' + required: true + responses: + 200: + $ref: '#/components/responses/S3SignResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 419: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + + ############################## + # Application Schema Objects # + ############################## +components: + schemas: + + S3Headers: + type: object + additionalProperties: + type: array + items: + type: string + + S3SignRequest: + required: + - region + - uri + - method + - headers + properties: + region: + type: string + uri: + type: string + method: + type: string + enum: ["PUT", "GET", "HEAD", "POST", "DELETE", "PATCH", "OPTIONS"] + headers: + $ref: '#/components/schemas/S3Headers' + properties: + type: object + additionalProperties: + type: string + body: + type: string + description: Optional body of the S3 request to send to the signing API. This should only be populated + for S3 requests which do not have the relevant data in the URI itself (e.g. DeleteObjects requests) + + + ############################# + # Reusable Response Objects # + ############################# + responses: + + S3SignResponse: + description: The response containing signed & unsigned headers. The server will also send + a Cache-Control header, indicating whether the response can be cached (Cache-Control = ["private"]) + or not (Cache-Control = ["no-cache"]). + content: + application/json: + schema: + type: object + required: + - uri + - headers + properties: + uri: + type: string + headers: + $ref: '#/components/schemas/S3Headers' diff --git a/spec/s3-sign/polaris-s3-sign-service.yaml b/spec/s3-sign/polaris-s3-sign-service.yaml new file mode 100644 index 0000000000..7e37b44117 --- /dev/null +++ b/spec/s3-sign/polaris-s3-sign-service.yaml @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Polaris-specific S3 Signer API that includes path parameters required for Polaris, such as +# the prefix, namespace, and table. + +openapi: 3.0.3 +info: + title: Apache Polaris S3 Signer API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the S3 Signer API integrated with Polaris catalog structure. + +# The server configuration is sourced from iceberg-s3-signer-open-api.yaml +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be appended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + +# All routes are currently configured using an Authorization header. +security: + - OAuth2: [] + +paths: + + /v1/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/prefix' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/namespace' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/table' + + post: + tags: + - S3 Signer API + summary: Remotely signs S3 requests + operationId: signS3Request + requestBody: + description: The request containing the headers to be signed + content: + application/json: + schema: + $ref: './iceberg-s3-signer-open-api.yaml#/components/schemas/S3SignRequest' + required: true + responses: + 200: + $ref: './iceberg-s3-signer-open-api.yaml#/components/responses/S3SignResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 419: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + +components: + securitySchemes: + OAuth2: + type: oauth2 + description: Uses OAuth 2 with client credentials flow + flows: + clientCredentials: + tokenUrl: "{scheme}://{host}/api/v1/oauth/tokens" + scopes: {} \ No newline at end of file From 4ea20e8788e0ddb13ec0d06ae27cf419cf6a49a0 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 14:55:33 +0100 Subject: [PATCH 02/10] add comment --- spec/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/README.md b/spec/README.md index 1bc01c21fc..567346a165 100644 --- a/spec/README.md +++ b/spec/README.md @@ -47,7 +47,8 @@ Apache Polaris provides the following OpenAPI specifications: - [s3-sign](s3-sign) - This folder contains the specifications for S3 remote signing: - [iceberg-s3-signer-open-api.yaml](s3-sign/iceberg-s3-signer-open-api.yaml) - Contains the specification of the - original Apache Iceberg S3 Signer API. + original Apache Iceberg S3 Signer API. Only the type definitions from this file are used; the actual API + endpoints are not used as Polaris uses custom endpoints (see next file). - [polaris-s3-sign-service.yaml](s3-sign/polaris-s3-sign-service.yaml) - Contains the Apache Polaris-specific S3 Signer API. From e3e19b3d5fc49dc1f29d7903e47da6b4217e74d6 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 14:55:59 +0100 Subject: [PATCH 03/10] don't advertise sign endpoint --- .../apache/polaris/core/rest/PolarisEndpoints.java | 12 ------------ .../catalog/iceberg/IcebergCatalogAdapter.java | 3 --- 2 files changed, 15 deletions(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java index 5b8f85e0d0..244c108c66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java @@ -99,16 +99,4 @@ public static Set getSupportedPolicyEndpoints(RealmConfig realmConfig) boolean policyStoreEnabled = realmConfig.getConfig(FeatureConfiguration.ENABLE_POLICY_STORE); return policyStoreEnabled ? POLICY_STORE_ENDPOINTS : ImmutableSet.of(); } - - /** - * Get the remote signing endpoints. Returns {@link #REMOTE_SIGNING_ENDPOINTS} if {@link - * FeatureConfiguration#REMOTE_SIGNING_ENABLED} is set globally to true or if the catalog enables - * remote signing; otherwise, returns an empty set. - */ - public static Set getSupportedRemoteSigningEndpoints( - RealmConfig realmConfig, CatalogEntity catalogEntity) { - boolean remoteSigningEnabled = - realmConfig.getConfig(FeatureConfiguration.REMOTE_SIGNING_ENABLED, catalogEntity); - return remoteSigningEnabled ? REMOTE_SIGNING_ENDPOINTS : ImmutableSet.of(); - } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index bdb7286ab2..034d66144e 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -830,9 +830,6 @@ public Response getConfig( .addAll(VIEW_ENDPOINTS) .addAll(PolarisEndpoints.getSupportedGenericTableEndpoints(realmConfig)) .addAll(PolarisEndpoints.getSupportedPolicyEndpoints(realmConfig)) - .addAll( - PolarisEndpoints.getSupportedRemoteSigningEndpoints( - callContext.getRealmConfig(), catalogEntity)) .build()) .build()) .build(); From fc30447bd756709d68c16f956e53f1e95f6191a7 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 14:56:32 +0100 Subject: [PATCH 04/10] api path segment constant --- .../org/apache/polaris/core/rest/PolarisResourcePaths.java | 6 ++++++ .../polaris/service/catalog/io/AccessConfigProvider.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java index bea8544bf1..34a7c622a4 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java @@ -28,6 +28,12 @@ public class PolarisResourcePaths { private static final Joiner SLASH = Joiner.on("/").skipNulls(); public static final String PREFIX = "prefix"; + /** + * The "api/" path segment is the first path segment of all Polaris and Iceberg REST API paths. It + * is not included in the constants below, as it is considered implicit. + */ + public static final String API_PATH_SEGMENT = "api"; + // Generic Table endpoints public static final String V1_GENERIC_TABLES = "polaris/v1/{prefix}/namespaces/{namespace}/generic-tables"; diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java index ad81a20871..a533a205ef 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java @@ -169,7 +169,7 @@ public AccessConfig getAccessConfigForRemoteSigning( } String prefix = prefixParser.catalogNameToPrefix(callContext.getRealmContext(), catalogName); - URI signerUri = uriInfo.getBaseUri().resolve("api/"); + URI signerUri = uriInfo.getBaseUriBuilder().path(PolarisResourcePaths.API_PATH_SEGMENT).build(); String signerEndpoint = new PolarisResourcePaths(prefix).s3RemoteSigning(tableIdentifier); return awsCredentialsStorageIntegration.getRemoteSigningAccessConfig(signerUri, signerEndpoint); From fd9d2ce41dbab900937dcb573a23d751327614d5 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 15:14:52 +0100 Subject: [PATCH 05/10] dont include credentials --- .../catalog/iceberg/IcebergCatalogHandler.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index 07dd97c6d4..d8da74e695 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -815,17 +815,6 @@ private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredential accessConfigProvider.getAccessConfigForRemoteSigning( callContext, catalogName, tableIdentifier, resolvedStoragePath); - Map credentialConfig = accessConfig.credentials(); - - if (!credentialConfig.isEmpty()) { - responseBuilder.addAllConfig(credentialConfig); - responseBuilder.addCredential( - ImmutableCredential.builder() - .prefix(tableMetadata.location()) - .config(credentialConfig) - .build()); - } - responseBuilder.addAllConfig(accessConfig.extraProperties()); return responseBuilder; From ca72870d63716bc08630bed15d034d540d2c78dd Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 15:15:06 +0100 Subject: [PATCH 06/10] improve access delegation mode selection logic --- .../iceberg/IcebergCatalogHandler.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index d8da74e695..ed535227a8 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -21,6 +21,7 @@ import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING; import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED; import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; +import static org.apache.polaris.service.catalog.AccessDelegationMode.UNKNOWN; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import com.google.common.base.Preconditions; @@ -866,13 +867,27 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { return responseBuilder; } - private static AccessDelegationMode selectAccessDelegationMode( + private AccessDelegationMode selectAccessDelegationMode( Set delegationModes) { + + // Whether vending credentials is globally enabled + boolean skipCredIndirection = + realmConfig.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION); + + // Credential subscoping is only allowed for local catalogs + // and federated catalogs that have credential vending explicitly enabled. + boolean credentialSubscopingAllowed = + baseCatalog instanceof IcebergCatalog + || realmConfig.getConfig( + ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity()); + + // Always prefer VENDED_CREDENTIALS if requested and available, + // even if REMOTE_SIGNING is also requested. return delegationModes.contains(VENDED_CREDENTIALS) - ? AccessDelegationMode.VENDED_CREDENTIALS - : delegationModes.contains(REMOTE_SIGNING) - ? AccessDelegationMode.REMOTE_SIGNING - : AccessDelegationMode.UNKNOWN; + && credentialSubscopingAllowed + && !skipCredIndirection + ? VENDED_CREDENTIALS + : delegationModes.contains(REMOTE_SIGNING) ? REMOTE_SIGNING : UNKNOWN; } private void validateRemoteTableLocations( From e025110c1eda74d307394a1b46b5d7ee6a50a997 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 15:17:44 +0100 Subject: [PATCH 07/10] spotless --- .../main/java/org/apache/polaris/core/rest/PolarisEndpoints.java | 1 - 1 file changed, 1 deletion(-) diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java index 244c108c66..71af8dd3e3 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisEndpoints.java @@ -23,7 +23,6 @@ import org.apache.iceberg.rest.Endpoint; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.config.RealmConfig; -import org.apache.polaris.core.entity.CatalogEntity; public class PolarisEndpoints { // Generic table endpoints From f843555aa1a886ec37a3415cc8f6524e19017a92 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 7 Nov 2025 16:18:31 +0100 Subject: [PATCH 08/10] fix selection algorithm --- .../iceberg/IcebergCatalogHandler.java | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index ed535227a8..c417d18055 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -870,24 +870,32 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { private AccessDelegationMode selectAccessDelegationMode( Set delegationModes) { - // Whether vending credentials is globally enabled - boolean skipCredIndirection = - realmConfig.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION); - - // Credential subscoping is only allowed for local catalogs - // and federated catalogs that have credential vending explicitly enabled. - boolean credentialSubscopingAllowed = - baseCatalog instanceof IcebergCatalog - || realmConfig.getConfig( - ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity()); - - // Always prefer VENDED_CREDENTIALS if requested and available, - // even if REMOTE_SIGNING is also requested. - return delegationModes.contains(VENDED_CREDENTIALS) - && credentialSubscopingAllowed - && !skipCredIndirection - ? VENDED_CREDENTIALS - : delegationModes.contains(REMOTE_SIGNING) ? REMOTE_SIGNING : UNKNOWN; + if (delegationModes.isEmpty()) { + return UNKNOWN; + } + + if (delegationModes.size() == 1) { + return delegationModes.iterator().next(); + } + + if (delegationModes.contains(VENDED_CREDENTIALS) && delegationModes.contains(REMOTE_SIGNING)) { + + boolean skipCredIndirection = + realmConfig.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION); + + boolean credentialSubscopingAllowed = + baseCatalog instanceof IcebergCatalog + || realmConfig.getConfig( + ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity()); + + // If both modes are supported, prefer VENDED_CREDENTIALS, + // but only if credential subscoping is allowed for this catalog + return !skipCredIndirection && credentialSubscopingAllowed + ? VENDED_CREDENTIALS + : REMOTE_SIGNING; + } + + throw new IllegalArgumentException("Unsupported access delegation modes: " + delegationModes); } private void validateRemoteTableLocations( @@ -1291,15 +1299,14 @@ private void checkAllowExternalCatalogCredentialVending() { CatalogEntity catalogEntity = getResolvedCatalogEntity(); LOGGER.info("Catalog type: {}", catalogEntity.getCatalogType()); - LOGGER.info( - "allow external catalog credential vending: {}", + Boolean allowCredentialVending = realmConfig.getConfig( - FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity)); - if (catalogEntity + FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity); + LOGGER.info("allow external catalog credential vending: {}", allowCredentialVending); + if (!allowCredentialVending + && catalogEntity .getCatalogType() - .equals(org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL) - && !realmConfig.getConfig( - FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity)) { + .equals(org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL)) { throw new ForbiddenException( "Access Delegation is not enabled for this catalog. Please consult applicable " + "documentation for the catalog config property '%s' to enable this feature", From f3961b05f587e30e25b068c96a1061b39039f885 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 10 Nov 2025 17:50:35 -0300 Subject: [PATCH 09/10] review --- CHANGELOG.md | 4 +-- .../iceberg/IcebergCatalogHandler.java | 9 ++++++ .../catalog/io/AccessConfigProvider.java | 1 + .../config/ProductionReadinessChecks.java | 28 +++++++++++++++++++ .../sign/S3RemoteSigningCatalogHandler.java | 8 +++++- .../S3RemoteSigningMinIOIntegrationTest.java | 1 + 6 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3de251340..d139d650c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,8 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti production-ready. One new table privilege was introduced: `TABLE_REMOTE_SIGN`. To enable remote signing: 1. Set the system-wide property `REMOTE_SIGNING_ENABLED` or the catalog-level `polaris.request-signing.enabled` property to `true`. - 2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The role must also be granted the `TABLE_READ_DATA` - and `TABLE_WRITE_DATA` privileges. + 2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The catalog role must also be granted the + `TABLE_READ_DATA` and `TABLE_WRITE_DATA` privileges. ### Upgrade notes diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index c417d18055..2a48639115 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -867,6 +867,14 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { return responseBuilder; } + /** + * Selects the most appropriate access delegation mode from the set of modes requested by the + * client. + * + *

See Iceberg + * REST Catalog spec. + */ private AccessDelegationMode selectAccessDelegationMode( Set delegationModes) { @@ -875,6 +883,7 @@ private AccessDelegationMode selectAccessDelegationMode( } if (delegationModes.size() == 1) { + // No need to validate the mode here, it will be validated later. return delegationModes.iterator().next(); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java index a533a205ef..d2f1a6da49 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/AccessConfigProvider.java @@ -169,6 +169,7 @@ public AccessConfig getAccessConfigForRemoteSigning( } String prefix = prefixParser.catalogNameToPrefix(callContext.getRealmContext(), catalogName); + // TODO M2 handle cases where the catalog server is behind a proxy URI signerUri = uriInfo.getBaseUriBuilder().path(PolarisResourcePaths.API_PATH_SEGMENT).build(); String signerEndpoint = new PolarisResourcePaths(prefix).s3RemoteSigning(tableIdentifier); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java index c1a03a7ab4..93cd66220d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java @@ -383,4 +383,32 @@ public ProductionReadinessCheck checkConnectionCredentialVendors( ? ProductionReadinessCheck.OK : ProductionReadinessCheck.of(errors.toArray(new Error[0])); } + + @Produces + public ProductionReadinessCheck checkRemoteSigning(FeaturesConfiguration featureConfiguration) { + var errors = new ArrayList(); + var offending = FeatureConfiguration.REMOTE_SIGNING_ENABLED; + if (Boolean.parseBoolean(featureConfiguration.defaults().get(offending.key()))) { + errors.add( + Error.ofSevere( + "Remote signing is an experimental feature and should not be enabled in production.", + format("polaris.features.\"%s\"", offending.key()))); + } + featureConfiguration + .realmOverrides() + .forEach( + (realmId, overrides) -> { + if (Boolean.parseBoolean(overrides.overrides().get(offending.key()))) { + errors.add( + Error.ofSevere( + "Remote signing is an experimental feature and should not be enabled in production.", + format( + "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", + realmId, offending.key()))); + } + }); + return errors.isEmpty() + ? ProductionReadinessCheck.OK + : ProductionReadinessCheck.of(errors.toArray(new Error[0])); + } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java index 26d668826a..86a645d807 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java @@ -128,6 +128,7 @@ public PolarisS3SignResponse signS3Request( return s3SignResponse; } + // TODO M2 computing allowed locations is expensive. We should cache it. private Collection getAllowedLocations(TableIdentifier tableIdentifier) { if (baseCatalog.tableExists(tableIdentifier)) { @@ -166,7 +167,12 @@ private Collection getAllowedLocations(TableIdentifier tableIdentifier) } private Set getStorageActions(PolarisS3SignRequest s3SignRequest) { - // TODO M2: better handling of DELETE and LIST + // TODO M2: better mapping of request URIs to storage actions. + // Disambiguate LIST vs READ or WRITE vs DELETE is not possible based on the HTTP method alone. + // Examples: + // - ListObjects is conceptually a LIST operation, and GetObject is conceptually a READ. But + // both requests use the GET method. + // - DeleteObject uses the DELETE method, but the DeleteObjects operation uses the POST method. return s3SignRequest.write() ? Set.of(PolarisStorageActions.WRITE) : Set.of(PolarisStorageActions.READ); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java index a3968a8b45..35739940e9 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java @@ -52,6 +52,7 @@ public Map getConfigOverrides() { return ImmutableMap.builder() .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.readiness.ignore-severe-issues", "true") .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]") .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") From 6019f321ea4ef912e5b291beaa42a1226da23c0b Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 10 Nov 2025 17:59:55 -0300 Subject: [PATCH 10/10] fix compilation failure --- .../service/storage/s3/sign/S3RemoteSigningCatalogHandler.java | 3 +-- .../s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java index 86a645d807..cce8805344 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java @@ -92,8 +92,7 @@ protected void initializeCatalog() { if (catalogEntity.isExternal()) { throw new ForbiddenException("Cannot use S3 remote signing with federated catalogs."); } - baseCatalog = - catalogFactory.createCallContextCatalog(callContext, polarisPrincipal, resolutionManifest); + baseCatalog = catalogFactory.createCallContextCatalog(resolutionManifest); } public PolarisS3SignResponse signS3Request( diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java index c35bdf9cd2..36147332ea 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java @@ -100,7 +100,7 @@ private S3RemoteSigningCatalogHandler newHandler() { .thenReturn(ImmutablePolarisS3SignResponse.builder().uri(URI.create("irrelevant")).build()); CallContextCatalogFactory callContextCatalogFactory = Mockito.mock(CallContextCatalogFactory.class); - Mockito.when(callContextCatalogFactory.createCallContextCatalog(any(), any(), any())) + Mockito.when(callContextCatalogFactory.createCallContextCatalog(any())) .thenReturn(baseCatalog); return new S3RemoteSigningCatalogHandler( diagServices,