diff --git a/CHANGELOG.md b/CHANGELOG.md index 493e0c60db..d139d650c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,14 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti [Iceberg Metrics Reporting]: https://iceberg.apache.org/docs/latest/metrics-reporting/ +- **S3 remote request signing** has been added, allowing Polaris to work with S3-compatible object storage systems. + *Remote signing is currently experimental and not enabled by default*. In particular, RBAC checks are currently not + production-ready. One new table privilege was introduced: `TABLE_REMOTE_SIGN`. To enable remote signing: + 1. Set the system-wide property `REMOTE_SIGNING_ENABLED` or the catalog-level `polaris.request-signing.enabled` + property to `true`. + 2. Grant the `TABLE_REMOTE_SIGN` privilege to a catalog role. The catalog role must also be granted the + `TABLE_READ_DATA` and `TABLE_WRITE_DATA` privileges. + ### Upgrade notes - The legacy management endpoints at `/metrics` and `/healthcheck` have been removed. Please use the diff --git a/LICENSE b/LICENSE index 8f50538f8d..2104ef28c6 100644 --- a/LICENSE +++ b/LICENSE @@ -217,6 +217,7 @@ This product includes code from Apache Iceberg. * spec/iceberg-rest-catalog-open-api.yaml * spec/polaris-catalog-apis/oauth-tokens-api.yaml +* spec/s3-sign/iceberg-s3-signer-open-api.yaml * integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java * runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java diff --git a/README.md b/README.md index 3245a4b086..fd6a1fb762 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ Apache Polaris is organized into the following modules: - `polaris-api-management-service` - Polaris Management API service classes - `polaris-api-iceberg-service` - The Iceberg REST service classes - `polaris-api-catalog-service` - The Polaris Catalog API service classes - - Runtime modules: + - `polaris-api-s3-sign-service` - The Iceberg REST service for S3 remote signing +- Runtime modules: - [`polaris-admin`](./runtime/admin/README.md) - The Polaris Admin Tool; mainly for bootstrapping persistence - [`polaris-runtime-defaults`](./runtime/defaults/README.md) - The runtime configuration defaults - [`polaris-distribution`](./runtime/distribution/README.md) - The Polaris distribution @@ -97,7 +98,6 @@ In addition to modules, there are: Outside of this repository, there are several other tools that can be found in a separate [Polaris-Tools](https://github.com/apache/polaris-tools) repository. ## Building and Running - Apache Polaris is built using Gradle with Java 21+ and Docker 27+. - `./gradlew build` - To build and run tests. Make sure Docker is running, as the integration tests depend on it. diff --git a/api/README.md b/api/README.md index 7c7fb61fc0..d8093d99be 100644 --- a/api/README.md +++ b/api/README.md @@ -33,6 +33,8 @@ This directory contains the API modules for Apache Polaris. Iceberg REST API. - [`polaris-api-catalog-service`](polaris-catalog-service): contains the service classes for the Polaris native Catalog REST API. +- [`polaris-api-s3-sign-service`](s3-sign-service): contains the model and service classes + for the S3 remote signing REST API. The classes in these modules are generated from the OpenAPI specification files in the [`spec`](../spec) directory. \ No newline at end of file diff --git a/api/s3-sign-service/build.gradle.kts b/api/s3-sign-service/build.gradle.kts new file mode 100644 index 0000000000..c3efbb14bb --- /dev/null +++ b/api/s3-sign-service/build.gradle.kts @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + alias(libs.plugins.openapi.generator) + id("polaris-client") + id("org.kordamp.gradle.jandex") +} + +dependencies { + implementation(project(":polaris-core")) + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + implementation("org.apache.iceberg:iceberg-aws") + + implementation(libs.jakarta.annotation.api) + implementation(libs.jakarta.inject.api) + implementation(libs.jakarta.validation.api) + implementation(libs.swagger.annotations) + + implementation(libs.jakarta.servlet.api) + implementation(libs.jakarta.ws.rs.api) + + implementation(platform(libs.micrometer.bom)) + implementation("io.micrometer:micrometer-core") + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.core:jackson-databind") + + compileOnly(libs.microprofile.fault.tolerance.api) + + compileOnly(project(":polaris-immutables")) + annotationProcessor(project(":polaris-immutables", configuration = "processor")) +} + +val rootDir = rootProject.layout.projectDirectory +val specsDir = rootDir.dir("spec") +val templatesDir = rootDir.dir("server-templates") +// Use a different directory than 'generated/', because OpenAPI generator's `GenerateTask` adds the +// whole directory to its task output, but 'generated/' is not exclusive to that task and in turn +// breaks Gradle's caching. +val generatedDir = project.layout.buildDirectory.dir("generated-openapi") +val generatedOpenApiSrcDir = project.layout.buildDirectory.dir("generated-openapi/src/main/java") + +openApiGenerate { + // The OpenAPI generator does NOT resolve relative paths correctly against the Gradle project + // directory + inputSpec = provider { specsDir.file("s3-sign/polaris-s3-sign-service.yaml").asFile.absolutePath } + generatorName = "jaxrs-resteasy" + outputDir = provider { generatedDir.get().asFile.absolutePath } + apiPackage = "org.apache.polaris.service.s3.sign.api" + ignoreFileOverride.set(provider { rootDir.file(".openapi-generator-ignore").asFile.absolutePath }) + templateDir.set(provider { templatesDir.asFile.absolutePath }) + removeOperationIdPrefix.set(true) + globalProperties.put("apis", "S3SignerApi") + globalProperties.put("models", "false") + globalProperties.put("apiDocs", "false") + globalProperties.put("modelTests", "false") + configOptions.put("resourceName", "catalog") + configOptions.put("useTags", "true") + configOptions.put("useBeanValidation", "false") + configOptions.put("sourceFolder", "src/main/java") + configOptions.put("useJakartaEe", "true") + configOptions.put("hideGenerationTimestamp", "true") + additionalProperties.put("apiNamePrefix", "IcebergRest") + additionalProperties.put("apiNameSuffix", "") + additionalProperties.put("metricsPrefix", "polaris") + serverVariables.put("basePath", "api/s3-sign") + modelNameMappings = mapOf("S3SignRequest" to "PolarisS3SignRequest") + typeMappings = + mapOf("S3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest") + importMappings = + mapOf( + "IcebergErrorResponse" to "org.apache.iceberg.rest.responses.ErrorResponse", + "PolarisS3SignRequest" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest", + "SignS3Request200Response" to "org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse", + ) +} + +listOf("sourcesJar", "compileJava", "processResources").forEach { task -> + tasks.named(task) { dependsOn("openApiGenerate") } +} + +sourceSets { main { java { srcDir(generatedOpenApiSrcDir) } } } + +tasks.named("openApiGenerate") { + inputs.dir(templatesDir) + inputs.dir(specsDir) + actions.addFirst { delete { delete(generatedDir) } } +} + +tasks.named("javadoc") { dependsOn("jandex") } diff --git a/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java new file mode 100644 index 0000000000..efc9d63746 --- /dev/null +++ b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignRequest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.s3.sign.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import jakarta.annotation.Nullable; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.immutables.PolarisImmutable; +import org.immutables.value.Value; + +/** + * Request for S3 signing. + * + *

Copy of {@link S3SignRequest}, because the original does not have Jackson annotations. + */ +@PolarisImmutable +@JsonDeserialize(as = ImmutablePolarisS3SignRequest.class) +@JsonSerialize(as = ImmutablePolarisS3SignRequest.class) +@SuppressWarnings("immutables:subtype") +public interface PolarisS3SignRequest extends S3SignRequest { + + @Value.Default + @Nullable // Replace javax.annotation.Nullable from S3SignRequest with jakarta.annotation.Nullable + @Override + default String body() { + return null; + } + + default boolean write() { + return method().equalsIgnoreCase("PUT") + || method().equalsIgnoreCase("POST") + || method().equalsIgnoreCase("DELETE") + || method().equalsIgnoreCase("PATCH"); + } +} diff --git a/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java new file mode 100644 index 0000000000..9f4cb778cf --- /dev/null +++ b/api/s3-sign-service/src/main/java/org/apache/polaris/service/s3/sign/model/PolarisS3SignResponse.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.s3.sign.model; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.polaris.immutables.PolarisImmutable; + +/** + * Response for S3 signing requests. + * + *

Copy of {@link S3SignResponse}, because the original does not have Jackson annotations. + */ +@PolarisImmutable +@JsonDeserialize(as = ImmutablePolarisS3SignResponse.class) +@JsonSerialize(as = ImmutablePolarisS3SignResponse.class) +@SuppressWarnings("immutables:subtype") +public interface PolarisS3SignResponse extends S3SignResponse {} diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index c959cfb766..1ec1e01e49 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { api(project(":polaris-api-iceberg-service")) api(project(":polaris-api-management-model")) api(project(":polaris-api-management-service")) + api(project(":polaris-api-s3-sign-service")) api(project(":polaris-container-spec-helper")) api(project(":polaris-minio-testcontainer")) diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 1b50c9ce4c..1c49673c74 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -24,6 +24,7 @@ polaris-api-iceberg-service=api/iceberg-service polaris-api-management-model=api/management-model polaris-api-management-service=api/management-service polaris-api-catalog-service=api/polaris-catalog-service +polaris-api-s3-sign-service=api/s3-sign-service polaris-runtime-defaults=runtime/defaults polaris-runtime-service=runtime/service polaris-server=runtime/server diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java new file mode 100644 index 0000000000..49be63bf7c --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisS3RemoteSigningIntegrationTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_ENDPOINT; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_PATH_STYLE_ACCESS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import com.google.common.collect.ImmutableMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.service.it.env.CatalogConfig; +import org.apache.polaris.service.it.env.RestCatalogConfig; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +/** Integration tests for S3 remote signing. */ +@CatalogConfig(properties = {"header.X-Iceberg-Access-Delegation", "remote-signing"}) +@RestCatalogConfig({ + // The default client file IO implementation is InMemoryFileIO, + // which does not support remote signing. + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.io.ResolvingFileIO", +}) +public abstract class PolarisS3RemoteSigningIntegrationTest + extends PolarisRestCatalogIntegrationBase { + + @Override + protected StorageConfigInfo getStorageConfigInfo() { + return AwsStorageConfigInfo.builder() + .setRoleArn(roleArn()) + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setPathStyleAccess(pathStyleAccess()) + .setAllowedLocations(allowedLocations()) + .setEndpoint(endpoint().orElse(null)) + .setStsUnavailable(stsUnavailable()) + .build(); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + ImmutableMap.Builder builder = + super.clientFileIOProperties() + .put(AWS_PATH_STYLE_ACCESS.getPropertyName(), String.valueOf(pathStyleAccess())); + endpoint().ifPresent(endpoint -> builder.put(AWS_ENDPOINT.getPropertyName(), endpoint)); + return builder; + } + + protected String roleArn() { + return "arn:aws:iam::123456789012:role/my-role"; + } + + protected boolean pathStyleAccess() { + return true; + } + + protected Optional endpoint() { + return Optional.empty(); + } + + /** + * A set of allowed locations to include in the {@linkplain #getStorageConfigInfo() storage + * configuration info}. The first allowed location will serve as the base for the catalog default + * location. + */ + protected abstract List allowedLocations(); + + protected boolean stsUnavailable() { + return false; + } + + @CatalogConfig(properties = {"polaris.config.remote-signing.enabled", "false"}) + @Test + public void testInternalCatalogRemoteSigningDisabled() { + @SuppressWarnings("resource") + RESTCatalog catalog = catalog(); + Namespace ns1 = Namespace.of("ns1"); + catalog.createNamespace(ns1); + TableIdentifier tableIdentifier = TableIdentifier.of(ns1, "my_table"); + assertThatThrownBy(() -> catalog.createTable(tableIdentifier, SCHEMA)) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Remote signing is not enabled for this catalog") + .hasMessageContaining(FeatureConfiguration.REMOTE_SIGNING_ENABLED.key()) + .hasMessageContaining(FeatureConfiguration.REMOTE_SIGNING_ENABLED.catalogConfig()); + } + + @CatalogConfig(Catalog.TypeEnum.EXTERNAL) + @Test + public void testExternalCatalogRemoteSigningDisabled() { + @SuppressWarnings("resource") + RESTCatalog catalog = catalog(); + Namespace ns1 = Namespace.of("ns1"); + catalog.createNamespace(ns1); + TableMetadata tableMetadata = + TableMetadata.newTableMetadata( + SCHEMA, + PartitionSpec.unpartitioned(), + externalCatalogBaseLocation() + "/ns1/my_table", + Map.of()); + try (ResolvingFileIO resolvingFileIO = initializeClientFileIO(new ResolvingFileIO())) { + String fileLocation = + externalCatalogBaseLocation() + "/ns1/my_table/metadata/v1.metadata.json"; + TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); + catalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); + try { + assertThatThrownBy(() -> catalog.loadTable(TableIdentifier.of(ns1, "my_table"))) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Remote signing is not enabled for external catalogs"); + } finally { + resolvingFileIO.deleteFile(fileLocation); + } + } + } + + @CatalogConfig( + properties = { + "polaris.config.default-table-location-object-storage-prefix.enabled", + "true", + "polaris.config.allow.overlapping.table.location", + "true" + }) + @Test + void testCreateTableWithObjectStoragePrefix() { + @SuppressWarnings("resource") + RESTCatalog restCatalog = catalog(); + restCatalog.createNamespace(NS); + // Only direct table creation is supported with object storage prefix + Table tbl1 = restCatalog.buildTable(TABLE, SCHEMA).create(); + // Will trigger write sign requests for manifests and snapshots, using object storage prefix + tbl1.newFastAppend().appendFile(FILE_A).commit(); + // Will trigger many read sign requests for metadata and manifests + assertFiles(tbl1, FILE_A); + assertThat(tbl1).isNotNull(); + } + + @CatalogConfig(properties = {"polaris.config.allow.unstructured.table.location", "true"}) + @Test + public void testCreateTableWithCustomLocation() { + @SuppressWarnings("resource") + RESTCatalog restCatalog = catalog(); + restCatalog.createNamespace(NS); + String customLocation = allowedLocations().getFirst() + "/custom/tbl1"; + Transaction create = + restCatalog.buildTable(TABLE, SCHEMA).withLocation(customLocation).createTransaction(); + // Will trigger write sign requests for manifests and snapshots, before the table is created + create.newFastAppend().appendFile(FILE_A).commit(); + // Will trigger table creation, then many read sign requests for metadata and manifests + create.commitTransaction(); + Table tbl1 = restCatalog.loadTable(TABLE); + assertFiles(tbl1, FILE_A); + assertThat(tbl1) + .isNotNull() + .asInstanceOf(type(BaseTable.class)) + .returns(customLocation, BaseTable::location); + } + + @Test + @Override + @Disabled("It's not possible to request an access delegation mode when registering a table.") + public void testRegisterTable() { + // FIXME this test should work if Polaris could send the right AccessConfig even if no + // delegation mode was requested when registering the table. + } + + @Test + @Override + @Disabled("This test is explicitly for vended credentials") + public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisabled() {} + + @Test + @Override + @Disabled("This test is explicitly for vended credentials") + public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisabled() {} +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java index 9d12cc148a..af642fe3d7 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizableOperation.java @@ -85,6 +85,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_GRANTS_ON_SECURABLE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOTE_SIGN; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_SNAPSHOTS; @@ -246,7 +247,10 @@ public enum PolarisAuthorizableOperation { REMOVE_TABLE_PROPERTIES(TABLE_REMOVE_PROPERTIES), SET_TABLE_STATISTICS(TABLE_SET_STATISTICS), REMOVE_TABLE_STATISTICS(TABLE_REMOVE_STATISTICS), - REMOVE_TABLE_PARTITION_SPECS(TABLE_REMOVE_PARTITION_SPECS); + REMOVE_TABLE_PARTITION_SPECS(TABLE_REMOVE_PARTITION_SPECS), + SIGN_S3_READ_REQUEST(EnumSet.of(TABLE_REMOTE_SIGN, TABLE_READ_DATA)), + SIGN_S3_WRITE_REQUEST(EnumSet.of(TABLE_REMOTE_SIGN, TABLE_WRITE_DATA)), + ; private final EnumSet privilegesOnTarget; private final EnumSet privilegesOnSecondary; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java index 489bc4fdae..68185b05e5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/auth/PolarisAuthorizerImpl.java @@ -99,6 +99,7 @@ import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_MANAGE_STRUCTURE; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_DATA; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_READ_PROPERTIES; +import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOTE_SIGN; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_PROPERTIES; import static org.apache.polaris.core.entity.PolarisPrivilege.TABLE_REMOVE_SNAPSHOTS; @@ -144,7 +145,7 @@ import org.slf4j.LoggerFactory; /** - * Performs hierarchical resolution logic by matching the transively expanded set of grants to a + * Performs hierarchical resolution logic by matching the transitively expanded set of grants to a * calling principal against the cascading permissions over the parent hierarchy of a target * Securable. * @@ -461,6 +462,10 @@ public class PolarisAuthorizerImpl implements PolarisAuthorizer { SUPER_PRIVILEGES.putAll( VIEW_FULL_METADATA, List.of(CATALOG_MANAGE_CONTENT, CATALOG_MANAGE_METADATA, VIEW_FULL_METADATA)); + SUPER_PRIVILEGES.putAll( + TABLE_REMOTE_SIGN, + List.of( + CATALOG_MANAGE_CONTENT, CATALOG_MANAGE_METADATA, TABLE_CREATE, TABLE_FULL_METADATA)); // Catalog privileges SUPER_PRIVILEGES.putAll( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java index 1772f47256..4c90feeb8d 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java @@ -438,4 +438,14 @@ public static void enforceFeatureEnabledOrThrow( "If set to true (default), allow credential vending for external catalogs. Note this requires ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING to be true first.") .defaultValue(true) .buildFeatureConfiguration(); + + public static final FeatureConfiguration REMOTE_SIGNING_ENABLED = + PolarisConfiguration.builder() + .key("REMOTE_SIGNING_ENABLED") + .catalogConfig("polaris.config.remote-signing.enabled") + .description( + "If true, the remote signing endpoints are enabled either globally, or for a specific catalog. " + + "This feature is currently experimental and may change in future releases.") + .defaultValue(false) + .buildFeatureConfiguration(); } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java index d76a6d457a..52f4a52f66 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/PolarisPrivilege.java @@ -245,6 +245,7 @@ public enum PolarisPrivilege { PolarisEntityType.TABLE_LIKE, List.of(PolarisEntitySubType.ICEBERG_TABLE, PolarisEntitySubType.GENERIC_TABLE), PolarisEntityType.CATALOG_ROLE), + TABLE_REMOTE_SIGN(103, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE), ; /** diff --git a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java index 16eea08da2..247e231a99 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/rest/PolarisResourcePaths.java @@ -28,6 +28,12 @@ public class PolarisResourcePaths { private static final Joiner SLASH = Joiner.on("/").skipNulls(); public static final String PREFIX = "prefix"; + /** + * The "api/" path segment is the first path segment of all Polaris and Iceberg REST API paths. It + * is not included in the constants below, as it is considered implicit. + */ + public static final String API_PATH_SEGMENT = "api"; + // Generic Table endpoints public static final String V1_GENERIC_TABLES = "polaris/v1/{prefix}/namespaces/{namespace}/generic-tables"; @@ -78,4 +84,15 @@ public String genericTable(TableIdentifier ident) { "generic-tables", RESTUtil.encodeString(ident.name())); } + + public String s3RemoteSigning(TableIdentifier ident) { + return SLASH.join( + "s3-sign", + "v1", + prefix, + "namespaces", + RESTUtil.encodeNamespace(ident.namespace()), + "tables", + RESTUtil.encodeString(ident.name())); + } } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java index ea4ca1c594..71b25e9118 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/InMemoryStorageIntegration.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.storage; import jakarta.annotation.Nonnull; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,7 +56,7 @@ protected InMemoryStorageIntegration(T config, String identifierOrId) { */ public static Map> validateAllowedLocations( @Nonnull RealmConfig realmConfig, - @Nonnull List allowedLocationsToValid, + @Nonnull Collection allowedLocationsToValid, @Nonnull Set actions, @Nonnull Set locations) { // trim trailing / from allowed locations so that locations missing the trailing slash still diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/LocationRestrictions.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/LocationRestrictions.java index 90ccf50d6d..947b9b0e62 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/LocationRestrictions.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/LocationRestrictions.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.storage; import jakarta.annotation.Nonnull; +import java.util.Collection; import java.util.List; import java.util.Set; import org.apache.iceberg.catalog.TableIdentifier; @@ -41,7 +42,7 @@ public class LocationRestrictions { *

All locations in this list have been validated to conform to the storage type's URI scheme * requirements during construction. */ - private final List allowedLocations; + private final Collection allowedLocations; /** * The parent location for structured table enforcement. @@ -54,15 +55,23 @@ public class LocationRestrictions { public LocationRestrictions( @Nonnull PolarisStorageConfigurationInfo storageConfigurationInfo, String parentLocation) { - this.allowedLocations = storageConfigurationInfo.getAllowedLocations(); + this(storageConfigurationInfo.getAllowedLocations(), parentLocation); allowedLocations.forEach(storageConfigurationInfo::validatePrefixForStorageType); - this.parentLocation = parentLocation; } public LocationRestrictions(@Nonnull PolarisStorageConfigurationInfo storageConfigurationInfo) { this(storageConfigurationInfo, null); } + public LocationRestrictions(Collection allowedLocations, String parentLocation) { + this.allowedLocations = allowedLocations; + this.parentLocation = parentLocation; + } + + public LocationRestrictions(Collection allowedLocations) { + this(allowedLocations, null); + } + /** * Validates that the requested storage locations are permitted for the given table identifier. * @@ -94,7 +103,7 @@ public void validate( private void validateLocations( RealmConfig realmConfig, - List allowedLocations, + Collection allowedLocations, Set requestLocations, TableIdentifier identifier) { var validationResults = diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java index 8edc8380d5..a5bc3b9a01 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java @@ -114,7 +114,7 @@ public static PolarisStorageConfigurationInfo deserialize(final @Nonnull String } } - public static Optional forEntityPath( + public static Optional getLocationRestrictionsForEntityPath( RealmConfig realmConfig, List entityPath) { return findStorageInfoFromHierarchy(entityPath) .map( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessConfig.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessConfig.java index 19745322d2..ddb74dc2bd 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessConfig.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessConfig.java @@ -23,10 +23,16 @@ import java.util.Map; import java.util.Optional; import org.apache.polaris.immutables.PolarisImmutable; -import org.immutables.value.Value; @PolarisImmutable public interface StorageAccessConfig { + + StorageAccessConfig EMPTY = + StorageAccessConfig.builder() + .supportsCredentialVending(false) + .supportsRemoteSigning(false) + .build(); + Map credentials(); Map extraProperties(); @@ -43,10 +49,13 @@ public interface StorageAccessConfig { * Indicates whether the storage integration subsystem that produced this object is capable of * credential vending in principle. */ - @Value.Default - default boolean supportsCredentialVending() { - return true; - } + boolean supportsCredentialVending(); + + /** + * Indicates whether the storage integration subsystem that produced this object is capable of + * remote signing in principle. + */ + boolean supportsRemoteSigning(); default String get(StorageAccessProperty key) { if (key.isCredential()) { @@ -77,6 +86,9 @@ interface Builder { @CanIgnoreReturnValue Builder supportsCredentialVending(boolean supportsCredentialVending); + @CanIgnoreReturnValue + Builder supportsRemoteSigning(boolean supportsRemoteSigning); + default Builder put(StorageAccessProperty key, String value) { if (key.isExpirationTimestamp()) { expiresAt(Instant.ofEpochMilli(Long.parseLong(value))); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java index 31a92c4f45..0ccc3ce0d6 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/StorageAccessProperty.java @@ -19,6 +19,8 @@ package org.apache.polaris.core.storage; import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; import org.apache.iceberg.azure.AzureProperties; import org.apache.iceberg.gcp.GCPProperties; @@ -52,6 +54,21 @@ public enum StorageAccessProperty { "the endpoint to load vended credentials for a table from the catalog", false, false), + AWS_REMOTE_SIGNING_ENABLED( + Boolean.class, + S3FileIOProperties.REMOTE_SIGNING_ENABLED, + "whether to enable remote signing for S3 requests", + false), + AWS_REMOTE_SIGNER_URI( + String.class, + S3V4RestSignerClient.S3_SIGNER_URI, + "the base URI for the remote signer service, used for signing S3 requests", + false), + AWS_REMOTE_SIGNER_ENDPOINT( + String.class, + S3V4RestSignerClient.S3_SIGNER_ENDPOINT, + "the endpoint for the remote signer service, used for signing S3 requests", + false), GCS_ACCESS_TOKEN(String.class, "gcs.oauth2.token", "the gcs scoped access token", true), GCS_ACCESS_TOKEN_EXPIRES_AT( diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index e393911f71..a047930a2e 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -80,7 +80,8 @@ public StorageAccessConfig getSubscopedCreds( realmConfig.getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS); AwsStorageConfigurationInfo storageConfig = config(); String region = storageConfig.getRegion(); - StorageAccessConfig.Builder accessConfig = StorageAccessConfig.builder(); + StorageAccessConfig.Builder accessConfig = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); if (shouldUseSts(storageConfig)) { AssumeRoleRequest.Builder request = @@ -120,6 +121,31 @@ public StorageAccessConfig getSubscopedCreds( }); } + addCommonProperties(region, accessConfig, storageConfig, refreshCredentialsEndpoint); + + return accessConfig.build(); + } + + public StorageAccessConfig getRemoteSigningAccessConfig(URI signerUri, String signerEndpoint) { + + AwsStorageConfigurationInfo storageConfig = config(); + StorageAccessConfig.Builder accessConfig = + StorageAccessConfig.builder().supportsCredentialVending(false).supportsRemoteSigning(true); + + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNING_ENABLED, Boolean.TRUE.toString()); + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNER_URI, signerUri.toString()); + accessConfig.put(StorageAccessProperty.AWS_REMOTE_SIGNER_ENDPOINT, signerEndpoint); + + addCommonProperties(storageConfig.getRegion(), accessConfig, storageConfig, Optional.empty()); + + return accessConfig.build(); + } + + private static void addCommonProperties( + String region, + StorageAccessConfig.Builder accessConfig, + AwsStorageConfigurationInfo storageConfig, + Optional refreshCredentialsEndpoint) { if (region != null) { accessConfig.put(StorageAccessProperty.CLIENT_REGION, region); } @@ -148,8 +174,6 @@ public StorageAccessConfig getSubscopedCreds( String.format( "AWS region must be set when using partition %s", storageConfig.getAwsPartition())); } - - return accessConfig.build(); } private boolean shouldUseSts(AwsStorageConfigurationInfo storageConfig) { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java index 0b189b3116..98e25942bf 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/azure/AzureCredentialsStorageIntegration.java @@ -181,7 +181,8 @@ static StorageAccessConfig toAccessConfig( String storageDnsName, Instant expiresAt, Optional refreshCredentialsEndpoint) { - StorageAccessConfig.Builder accessConfig = StorageAccessConfig.builder(); + StorageAccessConfig.Builder accessConfig = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); handleAzureCredential(accessConfig, sasToken, storageDnsName, expiresAt); accessConfig.put( StorageAccessProperty.EXPIRATION_TIME, String.valueOf(expiresAt.toEpochMilli())); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java index 5f524d9ae4..a65b1c00c9 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/gcp/GcpCredentialsStorageIntegration.java @@ -109,7 +109,8 @@ public StorageAccessConfig getSubscopedCreds( // If expires_in missing, use source credential's expire time, which require another api call to // get. - StorageAccessConfig.Builder accessConfig = StorageAccessConfig.builder(); + StorageAccessConfig.Builder accessConfig = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(false); accessConfig.put(StorageAccessProperty.GCS_ACCESS_TOKEN, token.getTokenValue()); accessConfig.put( StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, diff --git a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java index 14596911fd..3355e23c88 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/entity/PolarisPrivilegeTest.java @@ -131,7 +131,8 @@ static Stream polarisPrivileges() { Arguments.of(100, PolarisPrivilege.TABLE_REMOVE_STATISTICS), Arguments.of(101, PolarisPrivilege.TABLE_REMOVE_PARTITION_SPECS), Arguments.of(102, PolarisPrivilege.TABLE_MANAGE_STRUCTURE), - Arguments.of(103, null)); + Arguments.of(103, PolarisPrivilege.TABLE_REMOTE_SIGN), + Arguments.of(104, null)); } @ParameterizedTest diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageAccessConfigTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageAccessConfigTest.java index 98fad9ef9b..339b81183a 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageAccessConfigTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/StorageAccessConfigTest.java @@ -34,7 +34,8 @@ public class StorageAccessConfigTest { @Test public void testPutGet() { - StorageAccessConfig.Builder b = StorageAccessConfig.builder(); + StorageAccessConfig.Builder b = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.put(AWS_ENDPOINT, "ep1"); b.put(AWS_SECRET_KEY, "sk2"); StorageAccessConfig c = b.build(); @@ -46,7 +47,8 @@ public void testPutGet() { @Test public void testGetExtraProperty() { - StorageAccessConfig.Builder b = StorageAccessConfig.builder(); + StorageAccessConfig.Builder b = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra"); StorageAccessConfig c = b.build(); assertThat(c.extraProperties()).isEqualTo(Map.of(AWS_ENDPOINT.getPropertyName(), "extra")); @@ -55,7 +57,8 @@ public void testGetExtraProperty() { @Test public void testGetInternalProperty() { - StorageAccessConfig.Builder b = StorageAccessConfig.builder(); + StorageAccessConfig.Builder b = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.putExtraProperty(AWS_ENDPOINT.getPropertyName(), "extra"); b.putInternalProperty(AWS_ENDPOINT.getPropertyName(), "ep1"); StorageAccessConfig c = b.build(); @@ -66,7 +69,8 @@ public void testGetInternalProperty() { @Test public void testNoCredentialOverride() { - StorageAccessConfig.Builder b = StorageAccessConfig.builder(); + StorageAccessConfig.Builder b = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); b.put(AWS_SECRET_KEY, "sk-test"); b.putExtraProperty(AWS_SECRET_KEY.getPropertyName(), "sk-extra"); b.putInternalProperty(AWS_SECRET_KEY.getPropertyName(), "sk-internal"); @@ -79,7 +83,8 @@ public void testNoCredentialOverride() { @Test public void testExpiresAt() { - StorageAccessConfig.Builder b = StorageAccessConfig.builder(); + StorageAccessConfig.Builder b = + StorageAccessConfig.builder().supportsCredentialVending(true).supportsRemoteSigning(true); assertThat(b.build().expiresAt()).isEmpty(); b.put(GCS_ACCESS_TOKEN_EXPIRES_AT, "111"); assertThat(b.build().expiresAt()).hasValue(Instant.ofEpochMilli(111)); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java index c4db872317..7bc6c0e555 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/cache/StorageCredentialCacheTest.java @@ -372,6 +372,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( StorageAccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AWS_KEY_ID, "key_id_" + finalI) .put(StorageAccessProperty.AWS_SECRET_KEY, "key_secret_" + finalI) .put(StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, expireTime) @@ -381,6 +383,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( StorageAccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AZURE_SAS_TOKEN, "sas_token_" + finalI) .put(StorageAccessProperty.EXPIRATION_TIME, expireTime) .build())); @@ -388,6 +392,8 @@ private static List getFakeScopedCreds(int number, bool res.add( new ScopedCredentialsResult( StorageAccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.GCS_ACCESS_TOKEN, "gcs_token_" + finalI) .put(StorageAccessProperty.GCS_ACCESS_TOKEN_EXPIRES_AT, expireTime) .build())); @@ -418,6 +424,8 @@ public void testExtraProperties() { ScopedCredentialsResult properties = new ScopedCredentialsResult( StorageAccessConfig.builder() + .supportsCredentialVending(true) + .supportsRemoteSigning(true) .put(StorageAccessProperty.AWS_SECRET_KEY, "super-secret-123") .put(StorageAccessProperty.AWS_ENDPOINT, "test-endpoint1") .put(StorageAccessProperty.AWS_PATH_STYLE_ACCESS, "true") diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index c0f462982a..dda9c93680 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -30,6 +30,7 @@ dependencies { implementation(project(":polaris-api-management-service")) implementation(project(":polaris-api-iceberg-service")) implementation(project(":polaris-api-catalog-service")) + implementation(project(":polaris-api-s3-sign-service")) runtimeOnly(project(":polaris-relational-jdbc")) diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java new file mode 100644 index 0000000000..257eb63bb5 --- /dev/null +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIT.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.service.it.test.PolarisS3RemoteSigningIntegrationTest; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusIntegrationTest +@TestProfile(S3RemoteSigningMinIOIT.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class S3RemoteSigningMinIOIT extends PolarisS3RemoteSigningIntegrationTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static URI storageBase; + private static String endpoint; + + @BeforeAll + static void setup( + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess) { + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @Override + protected List allowedLocations() { + return List.of( + storageBase.resolve(BUCKET_URI_PREFIX + "/allowed-location1").toString(), + storageBase.resolve(BUCKET_URI_PREFIX + "/allowed-location2").toString()); + } + + @Override + protected Optional endpoint() { + return Optional.of(endpoint); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + return super.clientFileIOProperties() + // Grant direct access to the MinIO bucket; this FileIO instance does not + // use access delegation. + .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) + .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java index 7af8fac022..dee2b4f98a 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/CatalogPrefixParser.java @@ -18,28 +18,23 @@ */ package org.apache.polaris.service.catalog; -import org.apache.polaris.core.context.RealmContext; - /** An extension point for converting Iceberg REST API "prefix" values to Polaris Catalog names. */ public interface CatalogPrefixParser { /** - * Produces the name of a Polaris catalog from the given Iceberg Catalog REST API "prefix" for the - * specified realm. + * Produces the name of a Polaris catalog from the given Iceberg Catalog REST API "prefix". * - * @param realm identifies the realm where the API request is to be served. * @param prefix the "prefix" according to the Iceberg REST Catalog API specification. * @return Polaris Catalog name */ - String prefixToCatalogName(RealmContext realm, String prefix); + String prefixToCatalogName(String prefix); /** * Produces the "prefix" according to the Iceberg REST Catalog API specification for the given - * Polaris catalog name in the specified realm. + * Polaris catalog name. * - * @param realm identifies the realm owning the catalog * @param catalogName name of a Polaris catalog. * @return the "prefix" for the Iceberg REST client to be used for requests to the given catalog. */ - String catalogNameToPrefix(RealmContext realm, String catalogName); + String catalogNameToPrefix(String catalogName); } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java index 2573c9069f..aa5b74e732 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/DefaultCatalogPrefixParser.java @@ -19,17 +19,17 @@ package org.apache.polaris.service.catalog; import jakarta.enterprise.context.ApplicationScoped; -import org.apache.polaris.core.context.RealmContext; @ApplicationScoped public class DefaultCatalogPrefixParser implements CatalogPrefixParser { + @Override - public String prefixToCatalogName(RealmContext realm, String prefix) { + public String prefixToCatalogName(String prefix) { return prefix; } @Override - public String catalogNameToPrefix(RealmContext realm, String catalogName) { + public String catalogNameToPrefix(String catalogName) { return catalogName; } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java index 9d5778ac29..a35b39ebdd 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogHandler.java @@ -394,6 +394,47 @@ protected void authorizeRenameTableLikeOperationOrThrow( initializeCatalog(); } + protected void authorizeRemoteSigningOrThrow( + EnumSet ops, TableIdentifier identifier) { + + ensureResolutionManifestForTable(identifier); + PolarisResolvedPathWrapper target = + resolutionManifest.getResolvedPath( + identifier, PolarisEntityType.TABLE_LIKE, PolarisEntitySubType.ICEBERG_TABLE, true); + + // If the table doesn't exist, we still need to check allowed locations from the parent + // namespace. + if (target == null) { + resolutionManifest = newResolutionManifest(); + Namespace namespace = identifier.namespace(); + resolutionManifest.addPath( + new ResolverPath(Arrays.asList(namespace.levels()), PolarisEntityType.NAMESPACE), + namespace); + resolutionManifest.addPassthroughPath( + new ResolverPath( + PolarisCatalogHelpers.tableIdentifierToList(identifier), + PolarisEntityType.TABLE_LIKE, + true /* optional */), + identifier); + resolutionManifest.resolveAll(); + target = resolutionManifest.getResolvedPath(namespace, true); + if (target == null) { + throw new NoSuchNamespaceException("Namespace does not exist: %s", namespace); + } + } + + for (PolarisAuthorizableOperation op : ops) { + authorizer.authorizeOrThrow( + polarisPrincipal, + resolutionManifest.getAllActivatedCatalogRoleAndPrincipalRoles(), + op, + target, + null /* secondary */); + } + + initializeCatalog(); + } + /** * Helper function for when a TABLE_LIKE entity is not found so we want to throw the appropriate * exception. Used in Iceberg APIs, so the Iceberg messages cannot be changed. diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogUtils.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogUtils.java index ce82f36e37..ca2a8d582b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogUtils.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/common/CatalogUtils.java @@ -71,7 +71,7 @@ public static void validateLocationsForTableLike( Set locations, PolarisResolvedPathWrapper resolvedStorageEntity) { - PolarisStorageConfigurationInfo.forEntityPath( + PolarisStorageConfigurationInfo.getLocationRestrictionsForEntityPath( realmConfig, resolvedStorageEntity.getRawFullPath()) .ifPresentOrElse( restrictions -> restrictions.validate(realmConfig, identifier, locations), diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/CatalogGenericTableEventServiceDelegator.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/CatalogGenericTableEventServiceDelegator.java index e7bb6d925d..63a7ba8d8e 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/CatalogGenericTableEventServiceDelegator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/CatalogGenericTableEventServiceDelegator.java @@ -50,7 +50,7 @@ public Response createGenericTable( CreateGenericTableRequest createGenericTableRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeCreateGenericTable( new CatalogGenericTableServiceEvents.BeforeCreateGenericTableEvent( catalogName, namespace, createGenericTableRequest)); @@ -70,7 +70,7 @@ public Response dropGenericTable( String genericTable, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeDropGenericTable( new CatalogGenericTableServiceEvents.BeforeDropGenericTableEvent( catalogName, namespace, genericTable)); @@ -90,7 +90,7 @@ public Response listGenericTables( Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeListGenericTables( new CatalogGenericTableServiceEvents.BeforeListGenericTablesEvent(catalogName, namespace)); Response resp = @@ -108,7 +108,7 @@ public Response loadGenericTable( String genericTable, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeLoadGenericTable( new CatalogGenericTableServiceEvents.BeforeLoadGenericTableEvent( catalogName, namespace, genericTable)); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java index e52122dbcf..ba00e21a06 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/generic/GenericTableCatalogAdapter.java @@ -101,7 +101,7 @@ private GenericTableCatalogHandler newHandlerWrapper( resolutionManifestFactory, metaStoreManager, principal, - prefixParser.prefixToCatalogName(realmContext, prefix), + prefixParser.prefixToCatalogName(prefix), polarisAuthorizer, polarisCredentialManager, externalCatalogFactories); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java index e8f5402b1b..0ab8c7e578 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java @@ -2079,7 +2079,7 @@ private FileIO loadFileIOForTableLike( Map tableProperties, Set storageActions) { StorageAccessConfig storageAccessConfig = - storageAccessConfigProvider.getStorageAccessConfig( + storageAccessConfigProvider.getStorageAccessConfigForCredentialsVending( identifier, readLocations, storageActions, Optional.empty(), resolvedStorageEntity); // Reload fileIO based on table specific context FileIO fileIO = fileIOFactory.loadFileIO(storageAccessConfig, ioImplClassName, tableProperties); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java index 352d1a81cc..857e00c25b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogAdapter.java @@ -18,6 +18,7 @@ */ package org.apache.polaris.service.catalog.iceberg; +import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import static org.apache.polaris.service.catalog.validation.IcebergPropertiesValidation.validateIcebergProperties; @@ -35,6 +36,7 @@ import jakarta.ws.rs.core.SecurityContext; import java.util.EnumSet; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -67,7 +69,7 @@ import org.apache.polaris.core.context.CallContext; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.credentials.PolarisCredentialManager; -import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.ResolvedPolarisEntity; import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; @@ -198,7 +200,7 @@ private Response withCatalog( SecurityContext securityContext, String prefix, Function action) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); try (IcebergCatalogHandler wrapper = newHandlerWrapper(securityContext, catalogName)) { return action.apply(wrapper); } catch (RuntimeException e) { @@ -346,11 +348,13 @@ public Response updateProperties( catalog -> Response.ok(catalog.updateNamespaceProperties(ns, revisedRequest)).build()); } - private EnumSet parseAccessDelegationModes(String accessDelegationMode) { + private static Set parseAccessDelegationModes(String accessDelegationMode) { EnumSet delegationModes = AccessDelegationMode.fromProtocolValuesList(accessDelegationMode); Preconditions.checkArgument( - delegationModes.isEmpty() || delegationModes.contains(VENDED_CREDENTIALS), + delegationModes.isEmpty() + || delegationModes.contains(VENDED_CREDENTIALS) + || delegationModes.contains(REMOTE_SIGNING), "Unsupported access delegation mode: %s", accessDelegationMode); return delegationModes; @@ -365,8 +369,7 @@ public Response createTable( RealmContext realmContext, SecurityContext securityContext) { validateIcebergProperties(realmConfig, createTableRequest.properties()); - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); + Set delegationModes = parseAccessDelegationModes(accessDelegationMode); Namespace ns = decodeNamespace(namespace); return withCatalog( securityContext, @@ -418,8 +421,7 @@ public Response loadTable( String snapshots, RealmContext realmContext, SecurityContext securityContext) { - EnumSet delegationModes = - parseAccessDelegationModes(accessDelegationMode); + Set delegationModes = parseAccessDelegationModes(accessDelegationMode); Namespace ns = decodeNamespace(namespace); TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); @@ -451,9 +453,7 @@ public Response loadTable( } private static Optional getRefreshCredentialsEndpoint( - EnumSet delegationModes, - String prefix, - TableIdentifier tableIdentifier) { + Set delegationModes, String prefix, TableIdentifier tableIdentifier) { return delegationModes.contains(AccessDelegationMode.VENDED_CREDENTIALS) ? Optional.of(new PolarisResourcePaths(prefix).credentialsPath(tableIdentifier)) : Optional.empty(); @@ -621,6 +621,7 @@ public Response loadCredentials( catalog.loadTableWithAccessDelegation( tableIdentifier, "all", + EnumSet.of(VENDED_CREDENTIALS), Optional.of(new PolarisResourcePaths(prefix).credentialsPath(tableIdentifier))); return Response.ok( ImmutableLoadCredentialsResponse.builder() @@ -759,7 +760,7 @@ public Response reportMetrics( ReportMetricsRequest reportMetricsRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace ns = decodeNamespace(namespace); TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); @@ -814,10 +815,11 @@ public Response getConfig( throw new NotFoundException("Unable to find warehouse %s", warehouse); } ResolvedPolarisEntity resolvedReferenceCatalog = resolver.getResolvedReferenceCatalog(); - Map properties = - PolarisEntity.of(resolvedReferenceCatalog.getEntity()).getPropertiesAsMap(); + CatalogEntity catalogEntity = + CatalogEntity.of(Objects.requireNonNull(resolvedReferenceCatalog).getEntity()); + Map properties = catalogEntity.getPropertiesAsMap(); - String prefix = prefixParser.catalogNameToPrefix(realmContext, warehouse); + String prefix = prefixParser.catalogNameToPrefix(warehouse); return Response.ok( ConfigResponse.builder() .withDefaults(properties) // catalog properties are defaults diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java index d2e48d2cc0..4c8f5f903c 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandler.java @@ -20,6 +20,8 @@ import static org.apache.polaris.core.config.FeatureConfiguration.ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING; import static org.apache.polaris.core.config.FeatureConfiguration.LIST_PAGINATION_ENABLED; +import static org.apache.polaris.service.catalog.AccessDelegationMode.REMOTE_SIGNING; +import static org.apache.polaris.service.catalog.AccessDelegationMode.UNKNOWN; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import com.google.common.base.Preconditions; @@ -112,6 +114,7 @@ import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.apache.polaris.service.http.IcebergHttpUtil; import org.apache.polaris.service.http.IfNoneMatch; +import org.apache.polaris.service.storage.s3.sign.S3RemoteSigningCatalogHandler; import org.apache.polaris.service.types.NotificationRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -400,15 +403,13 @@ public LoadTableResponse createTableDirect(Namespace namespace, CreateTableReque public LoadTableResponse createTableDirectWithWriteDelegation( Namespace namespace, CreateTableRequest request, + Set delegationModes, Optional refreshCredentialsEndpoint) { - return createTableDirect( - namespace, request, EnumSet.of(VENDED_CREDENTIALS), refreshCredentialsEndpoint); + return createTableDirect(namespace, request, delegationModes, refreshCredentialsEndpoint); } public void authorizeCreateTableDirect( - Namespace namespace, - CreateTableRequest request, - EnumSet delegationModes) { + Namespace namespace, CreateTableRequest request, Set delegationModes) { if (delegationModes.isEmpty()) { TableIdentifier identifier = TableIdentifier.of(namespace, request.name()); authorizeCreateTableLikeUnderNamespaceOperationOrThrow( @@ -423,13 +424,16 @@ public void authorizeCreateTableDirect( if (catalog.isStaticFacade()) { throw new BadRequestException("Cannot create table on static-facade external catalogs."); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } } public LoadTableResponse createTableDirect( Namespace namespace, CreateTableRequest request, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { authorizeCreateTableDirect(namespace, request, delegationModes); @@ -526,15 +530,13 @@ public LoadTableResponse createTableStaged(Namespace namespace, CreateTableReque public LoadTableResponse createTableStagedWithWriteDelegation( Namespace namespace, CreateTableRequest request, + Set delegationModes, Optional refreshCredentialsEndpoint) { - return createTableStaged( - namespace, request, EnumSet.of(VENDED_CREDENTIALS), refreshCredentialsEndpoint); + return createTableStaged(namespace, request, delegationModes, refreshCredentialsEndpoint); } private void authorizeCreateTableStaged( - Namespace namespace, - CreateTableRequest request, - EnumSet delegationModes) { + Namespace namespace, CreateTableRequest request, Set delegationModes) { if (delegationModes.isEmpty()) { authorizeCreateTableLikeUnderNamespaceOperationOrThrow( PolarisAuthorizableOperation.CREATE_TABLE_STAGED, @@ -549,13 +551,16 @@ private void authorizeCreateTableStaged( if (catalog.isStaticFacade()) { throw new BadRequestException("Cannot create table on static-facade external catalogs."); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } } public LoadTableResponse createTableStaged( Namespace namespace, CreateTableRequest request, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { authorizeCreateTableStaged(namespace, request, delegationModes); @@ -662,9 +667,10 @@ public Optional loadTableIfStale( public LoadTableResponse loadTableWithAccessDelegation( TableIdentifier tableIdentifier, String snapshots, + Set delegationModes, Optional refreshCredentialsEndpoint) { return loadTableWithAccessDelegationIfStale( - tableIdentifier, null, snapshots, refreshCredentialsEndpoint) + tableIdentifier, null, snapshots, delegationModes, refreshCredentialsEndpoint) .get(); } @@ -682,17 +688,14 @@ public Optional loadTableWithAccessDelegationIfStale( TableIdentifier tableIdentifier, IfNoneMatch ifNoneMatch, String snapshots, + Set delegationModes, Optional refreshCredentialsEndpoint) { return loadTable( - tableIdentifier, - snapshots, - ifNoneMatch, - EnumSet.of(VENDED_CREDENTIALS), - refreshCredentialsEndpoint); + tableIdentifier, snapshots, ifNoneMatch, delegationModes, refreshCredentialsEndpoint); } private Set authorizeLoadTable( - TableIdentifier tableIdentifier, EnumSet delegationModes) { + TableIdentifier tableIdentifier, Set delegationModes) { if (delegationModes.isEmpty()) { authorizeBasicTableLikeOperationOrThrow( PolarisAuthorizableOperation.LOAD_TABLE, @@ -724,7 +727,10 @@ private Set authorizeLoadTable( read, PolarisEntitySubType.ICEBERG_TABLE, tableIdentifier); } - checkAllowExternalCatalogCredentialVending(delegationModes); + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + if (delegationMode == VENDED_CREDENTIALS) { + checkAllowExternalCatalogCredentialVending(); + } return actionsRequested; } @@ -733,7 +739,7 @@ public Optional loadTable( TableIdentifier tableIdentifier, String snapshots, IfNoneMatch ifNoneMatch, - EnumSet delegationModes, + Set delegationModes, Optional refreshCredentialsEndpoint) { Set actionsRequested = @@ -785,7 +791,7 @@ public Optional loadTable( private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredentials( TableIdentifier tableIdentifier, TableMetadata tableMetadata, - EnumSet delegationModes, + Set delegationModes, Set actions, Optional refreshCredentialsEndpoint) { LoadTableResponse.Builder responseBuilder = @@ -799,6 +805,22 @@ private LoadTableResponse.Builder buildLoadTableResponseWithDelegationCredential return responseBuilder; } + AccessDelegationMode delegationMode = selectAccessDelegationMode(delegationModes); + + if (delegationMode == REMOTE_SIGNING) { + + S3RemoteSigningCatalogHandler.throwIfRemoteSigningNotEnabled( + callContext.getRealmConfig(), getResolvedCatalogEntity()); + + StorageAccessConfig accessConfig = + storageAccessConfigProvider.getStorageAccessConfigForRemoteSigning( + catalogName, tableIdentifier, resolvedStoragePath); + + responseBuilder.addAllConfig(accessConfig.extraProperties()); + + return responseBuilder; + } + if (baseCatalog instanceof IcebergCatalog || realmConfig.getConfig( ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { @@ -811,13 +833,16 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { } StorageAccessConfig storageAccessConfig = - storageAccessConfigProvider.getStorageAccessConfig( + storageAccessConfigProvider.getStorageAccessConfigForCredentialsVending( tableIdentifier, tableLocations, actions, refreshCredentialsEndpoint, resolvedStoragePath); Map credentialConfig = storageAccessConfig.credentials(); + + // FIXME it would be cleaner to move this into the accessConfigProvider + // or create another method getAccessConfigNoDelegation() if (delegationModes.contains(VENDED_CREDENTIALS)) { if (!credentialConfig.isEmpty()) { responseBuilder.addAllConfig(credentialConfig); @@ -841,6 +866,46 @@ ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity())) { return responseBuilder; } + /** + * Selects the most appropriate access delegation mode from the set of modes requested by the + * client. + * + *

See Iceberg + * REST Catalog spec. + */ + private AccessDelegationMode selectAccessDelegationMode( + Set delegationModes) { + + if (delegationModes.isEmpty()) { + return UNKNOWN; + } + + if (delegationModes.size() == 1) { + // No need to validate the mode here, it will be validated later. + return delegationModes.iterator().next(); + } + + if (delegationModes.contains(VENDED_CREDENTIALS) && delegationModes.contains(REMOTE_SIGNING)) { + + boolean skipCredIndirection = + realmConfig.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION); + + boolean credentialSubscopingAllowed = + baseCatalog instanceof IcebergCatalog + || realmConfig.getConfig( + ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING, getResolvedCatalogEntity()); + + // If both modes are supported, prefer VENDED_CREDENTIALS, + // but only if credential subscoping is allowed for this catalog + return !skipCredIndirection && credentialSubscopingAllowed + ? VENDED_CREDENTIALS + : REMOTE_SIGNING; + } + + throw new IllegalArgumentException("Unsupported access delegation modes: " + delegationModes); + } + private void validateRemoteTableLocations( TableIdentifier tableIdentifier, Set tableLocations, @@ -1238,24 +1303,18 @@ private EnumSet getUpdateTableAuthorizableOperatio } } - private void checkAllowExternalCatalogCredentialVending( - EnumSet delegationModes) { - - if (delegationModes.isEmpty()) { - return; - } + private void checkAllowExternalCatalogCredentialVending() { CatalogEntity catalogEntity = getResolvedCatalogEntity(); LOGGER.info("Catalog type: {}", catalogEntity.getCatalogType()); - LOGGER.info( - "allow external catalog credential vending: {}", + Boolean allowCredentialVending = realmConfig.getConfig( - FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity)); - if (catalogEntity + FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity); + LOGGER.info("allow external catalog credential vending: {}", allowCredentialVending); + if (!allowCredentialVending + && catalogEntity .getCatalogType() - .equals(org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL) - && !realmConfig.getConfig( - FeatureConfiguration.ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING, catalogEntity)) { + .equals(org.apache.polaris.core.admin.model.Catalog.TypeEnum.EXTERNAL)) { throw new ForbiddenException( "Access Delegation is not enabled for this catalog. Please consult applicable " + "documentation for the catalog config property '%s' to enable this feature", diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRestCatalogEventServiceDelegator.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRestCatalogEventServiceDelegator.java index 2c9e3cbd87..d463331e6d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRestCatalogEventServiceDelegator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergRestCatalogEventServiceDelegator.java @@ -110,7 +110,7 @@ public Response createNamespace( CreateNamespaceRequest createNamespaceRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeCreateNamespace( new BeforeCreateNamespaceEvent(catalogName, createNamespaceRequest)); Response resp = @@ -132,7 +132,7 @@ public Response listNamespaces( String parent, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeListNamespaces(new BeforeListNamespacesEvent(catalogName, parent)); Response resp = delegate.listNamespaces(prefix, pageToken, pageSize, parent, realmContext, securityContext); @@ -143,7 +143,7 @@ public Response listNamespaces( @Override public Response loadNamespaceMetadata( String prefix, String namespace, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeLoadNamespaceMetadata( new BeforeLoadNamespaceMetadataEvent(catalogName, decodeNamespace(namespace))); Response resp = @@ -158,7 +158,7 @@ public Response loadNamespaceMetadata( @Override public Response namespaceExists( String prefix, String namespace, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeCheckExistsNamespace( new BeforeCheckExistsNamespaceEvent(catalogName, namespaceObj)); @@ -171,7 +171,7 @@ public Response namespaceExists( @Override public Response dropNamespace( String prefix, String namespace, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeDropNamespace( new BeforeDropNamespaceEvent(catalogName, decodeNamespace(namespace))); Response resp = delegate.dropNamespace(prefix, namespace, realmContext, securityContext); @@ -186,7 +186,7 @@ public Response updateProperties( UpdateNamespacePropertiesRequest updateNamespacePropertiesRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeUpdateNamespaceProperties( new BeforeUpdateNamespacePropertiesEvent( @@ -208,7 +208,7 @@ public Response createTable( String accessDelegationMode, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeCreateTable( new BeforeCreateTableEvent( @@ -240,7 +240,7 @@ public Response listTables( Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeListTables(new BeforeListTablesEvent(catalogName, namespaceObj)); Response resp = @@ -259,7 +259,7 @@ public Response loadTable( String snapshots, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeLoadTable( new BeforeLoadTableEvent( @@ -287,7 +287,7 @@ public Response tableExists( String table, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeCheckExistsTable( new BeforeCheckExistsTableEvent(catalogName, namespaceObj, table)); @@ -305,7 +305,7 @@ public Response dropTable( Boolean purgeRequested, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeDropTable( new BeforeDropTableEvent(catalogName, namespaceObj, table, purgeRequested)); @@ -323,7 +323,7 @@ public Response registerTable( RegisterTableRequest registerTableRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeRegisterTable( new BeforeRegisterTableEvent(catalogName, namespaceObj, registerTableRequest)); @@ -345,7 +345,7 @@ public Response renameTable( RenameTableRequest renameTableRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeRenameTable( new BeforeRenameTableEvent(catalogName, renameTableRequest)); Response resp = delegate.renameTable(prefix, renameTableRequest, realmContext, securityContext); @@ -362,7 +362,7 @@ public Response updateTable( CommitTableRequest commitTableRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeUpdateTable( new BeforeUpdateTableEvent(catalogName, namespaceObj, table, commitTableRequest)); @@ -386,7 +386,7 @@ public Response createView( CreateViewRequest createViewRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeCreateView( new BeforeCreateViewEvent(catalogName, namespaceObj, createViewRequest)); @@ -409,7 +409,7 @@ public Response listViews( Integer pageSize, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeListViews(new BeforeListViewsEvent(catalogName, namespaceObj)); Response resp = @@ -425,7 +425,7 @@ public Response loadCredentials( String table, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeLoadCredentials( new BeforeLoadCredentialsEvent(catalogName, namespaceObj, table)); @@ -443,7 +443,7 @@ public Response loadView( String view, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeLoadView(new BeforeLoadViewEvent(catalogName, namespaceObj, view)); Response resp = delegate.loadView(prefix, namespace, view, realmContext, securityContext); @@ -460,7 +460,7 @@ public Response viewExists( String view, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeCheckExistsView( new BeforeCheckExistsViewEvent(catalogName, namespaceObj, view)); @@ -477,7 +477,7 @@ public Response dropView( String view, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeDropView(new BeforeDropViewEvent(catalogName, namespaceObj, view)); Response resp = delegate.dropView(prefix, namespace, view, realmContext, securityContext); @@ -491,7 +491,7 @@ public Response renameView( RenameTableRequest renameTableRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeRenameView( new BeforeRenameViewEvent(catalogName, renameTableRequest)); Response resp = delegate.renameView(prefix, renameTableRequest, realmContext, securityContext); @@ -508,7 +508,7 @@ public Response replaceView( CommitViewRequest commitViewRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeReplaceView( new BeforeReplaceViewEvent(catalogName, namespaceObj, view, commitViewRequest)); @@ -531,7 +531,7 @@ public Response commitTransaction( CommitTransactionRequest commitTransactionRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeCommitTransaction( new IcebergRestCatalogEvents.BeforeCommitTransactionEvent( catalogName, commitTransactionRequest)); @@ -564,7 +564,7 @@ public Response sendNotification( NotificationRequest notificationRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); Namespace namespaceObj = decodeNamespace(namespace); polarisEventListener.onBeforeSendNotification( new BeforeSendNotificationEvent(catalogName, namespaceObj, table, notificationRequest)); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java index d6316c6e7a..8e121def19 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/io/StorageAccessConfigProvider.java @@ -22,16 +22,26 @@ import jakarta.annotation.Nonnull; import jakarta.enterprise.context.RequestScoped; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; +import java.net.URI; import java.util.Optional; import java.util.Set; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.polaris.core.config.FeatureConfiguration; import org.apache.polaris.core.entity.PolarisEntity; +import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.rest.PolarisResourcePaths; import org.apache.polaris.core.storage.PolarisStorageActions; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.apache.polaris.core.storage.PolarisStorageIntegration; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.StorageAccessConfig; import org.apache.polaris.core.storage.StorageCredentialsVendor; +import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; +import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,13 +59,22 @@ public class StorageAccessConfigProvider { private final StorageCredentialCache storageCredentialCache; private final StorageCredentialsVendor storageCredentialsVendor; + private final PolarisStorageIntegrationProvider storageIntegrationProvider; + private final CatalogPrefixParser prefixParser; + private final UriInfo uriInfo; @Inject public StorageAccessConfigProvider( StorageCredentialCache storageCredentialCache, - StorageCredentialsVendor storageCredentialsVendor) { + StorageCredentialsVendor storageCredentialsVendor, + PolarisStorageIntegrationProvider storageIntegrationProvider, + CatalogPrefixParser prefixParser, + UriInfo uriInfo) { this.storageCredentialCache = storageCredentialCache; this.storageCredentialsVendor = storageCredentialsVendor; + this.storageIntegrationProvider = storageIntegrationProvider; + this.prefixParser = prefixParser; + this.uriInfo = uriInfo; } /** @@ -70,7 +89,7 @@ public StorageAccessConfigProvider( * @return {@link StorageAccessConfig} with scoped credentials and metadata; empty if no storage * config found */ - public StorageAccessConfig getStorageAccessConfig( + public StorageAccessConfig getStorageAccessConfigForCredentialsVending( @Nonnull TableIdentifier tableIdentifier, @Nonnull Set tableLocations, @Nonnull Set storageActions, @@ -87,7 +106,7 @@ public StorageAccessConfig getStorageAccessConfig( .atWarn() .addKeyValue("tableIdentifier", tableIdentifier) .log("Table entity has no storage configuration in its hierarchy"); - return StorageAccessConfig.builder().supportsCredentialVending(false).build(); + return StorageAccessConfig.EMPTY; } PolarisEntity storageInfoEntity = storageInfo.get(); @@ -100,7 +119,7 @@ public StorageAccessConfig getStorageAccessConfig( .atDebug() .addKeyValue("tableIdentifier", tableIdentifier) .log("Skipping generation of subscoped creds for table"); - return StorageAccessConfig.builder().build(); + return StorageAccessConfig.EMPTY; } boolean allowList = @@ -132,4 +151,57 @@ public StorageAccessConfig getStorageAccessConfig( } return accessConfig; } + + /** + * Generates a remote signing configuration for accessing table storage at explicit locations. + * + * @param catalogName the name of the catalog + * @param tableIdentifier the table identifier, used for logging and refresh endpoint construction + * @param resolvedPath the entity hierarchy to search for storage configuration + * @return {@link StorageAccessConfig} with scoped credentials and metadata; empty if no storage + * config found + */ + public StorageAccessConfig getStorageAccessConfigForRemoteSigning( + @Nonnull String catalogName, + @Nonnull TableIdentifier tableIdentifier, + @Nonnull PolarisResolvedPathWrapper resolvedPath) { + LOGGER + .atDebug() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Fetching remote signing config for table"); + + Optional storageInfo = FileIOUtil.findStorageInfoFromHierarchy(resolvedPath); + Optional configurationInfo = + storageInfo + .map(PolarisEntity::getInternalPropertiesAsMap) + .map(info -> info.get(PolarisEntityConstants.getStorageConfigInfoPropertyName())) + .map(PolarisStorageConfigurationInfo::deserialize); + + if (configurationInfo.isEmpty()) { + LOGGER + .atWarn() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Table entity has no storage configuration in its hierarchy"); + return StorageAccessConfig.EMPTY; + } + + PolarisStorageIntegration storageIntegration = + storageIntegrationProvider.getStorageIntegrationForConfig(configurationInfo.get()); + + if (!(storageIntegration + instanceof AwsCredentialsStorageIntegration awsCredentialsStorageIntegration)) { + LOGGER + .atWarn() + .addKeyValue("tableIdentifier", tableIdentifier) + .log("Table entity storage integration is not an AWS credentials storage integration"); + return StorageAccessConfig.EMPTY; + } + + String prefix = prefixParser.catalogNameToPrefix(catalogName); + // TODO M2 handle cases where the catalog server is behind a proxy + URI signerUri = uriInfo.getBaseUriBuilder().path(PolarisResourcePaths.API_PATH_SEGMENT).build(); + String signerEndpoint = new PolarisResourcePaths(prefix).s3RemoteSigning(tableIdentifier); + + return awsCredentialsStorageIntegration.getRemoteSigningAccessConfig(signerUri, signerEndpoint); + } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/CatalogPolicyEventServiceDelegator.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/CatalogPolicyEventServiceDelegator.java index 2bf671db28..8b31e13eb9 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/CatalogPolicyEventServiceDelegator.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/CatalogPolicyEventServiceDelegator.java @@ -54,7 +54,7 @@ public Response createPolicy( CreatePolicyRequest createPolicyRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeCreatePolicy( new CatalogPolicyServiceEvents.BeforeCreatePolicyEvent( catalogName, namespace, createPolicyRequest)); @@ -76,7 +76,7 @@ public Response listPolicies( String policyType, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeListPolicies( new CatalogPolicyServiceEvents.BeforeListPoliciesEvent(catalogName, namespace, policyType)); Response resp = @@ -94,7 +94,7 @@ public Response loadPolicy( String policyName, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeLoadPolicy( new CatalogPolicyServiceEvents.BeforeLoadPolicyEvent(catalogName, namespace, policyName)); Response resp = @@ -113,7 +113,7 @@ public Response updatePolicy( UpdatePolicyRequest updatePolicyRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeUpdatePolicy( new CatalogPolicyServiceEvents.BeforeUpdatePolicyEvent( catalogName, namespace, policyName, updatePolicyRequest)); @@ -134,7 +134,7 @@ public Response dropPolicy( Boolean detachAll, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeDropPolicy( new CatalogPolicyServiceEvents.BeforeDropPolicyEvent( catalogName, namespace, policyName, detachAll)); @@ -155,7 +155,7 @@ public Response attachPolicy( AttachPolicyRequest attachPolicyRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeAttachPolicy( new CatalogPolicyServiceEvents.BeforeAttachPolicyEvent( catalogName, namespace, policyName, attachPolicyRequest)); @@ -176,7 +176,7 @@ public Response detachPolicy( DetachPolicyRequest detachPolicyRequest, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeDetachPolicy( new CatalogPolicyServiceEvents.BeforeDetachPolicyEvent( catalogName, namespace, policyName, detachPolicyRequest)); @@ -199,7 +199,7 @@ public Response getApplicablePolicies( String policyType, RealmContext realmContext, SecurityContext securityContext) { - String catalogName = prefixParser.prefixToCatalogName(realmContext, prefix); + String catalogName = prefixParser.prefixToCatalogName(prefix); polarisEventListener.onBeforeGetApplicablePolicies( new CatalogPolicyServiceEvents.BeforeGetApplicablePoliciesEvent( catalogName, namespace, targetName, policyType)); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java index 9bdfef4835..9b6ebd4e7b 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/catalog/policy/PolicyCatalogAdapter.java @@ -101,7 +101,7 @@ private PolicyCatalogHandler newHandlerWrapper(SecurityContext securityContext, resolutionManifestFactory, metaStoreManager, principal, - prefixParser.prefixToCatalogName(realmContext, prefix), + prefixParser.prefixToCatalogName(prefix), polarisAuthorizer, polarisCredentialManager, externalCatalogFactories); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java index c1a03a7ab4..93cd66220d 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/config/ProductionReadinessChecks.java @@ -383,4 +383,32 @@ public ProductionReadinessCheck checkConnectionCredentialVendors( ? ProductionReadinessCheck.OK : ProductionReadinessCheck.of(errors.toArray(new Error[0])); } + + @Produces + public ProductionReadinessCheck checkRemoteSigning(FeaturesConfiguration featureConfiguration) { + var errors = new ArrayList(); + var offending = FeatureConfiguration.REMOTE_SIGNING_ENABLED; + if (Boolean.parseBoolean(featureConfiguration.defaults().get(offending.key()))) { + errors.add( + Error.ofSevere( + "Remote signing is an experimental feature and should not be enabled in production.", + format("polaris.features.\"%s\"", offending.key()))); + } + featureConfiguration + .realmOverrides() + .forEach( + (realmId, overrides) -> { + if (Boolean.parseBoolean(overrides.overrides().get(offending.key()))) { + errors.add( + Error.ofSevere( + "Remote signing is an experimental feature and should not be enabled in production.", + format( + "polaris.features.realm-overrides.\"%s\".overrides.\"%s\"", + realmId, offending.key()))); + } + }); + return errors.isEmpty() + ? ProductionReadinessCheck.OK + : ProductionReadinessCheck.of(errors.toArray(new Error[0])); + } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 706acb4222..6e168a8d27 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -53,13 +53,12 @@ public class PolarisStorageIntegrationProviderImpl implements PolarisStorageInte private final Optional stsCredentials; private final Supplier gcpCredsProvider; - @SuppressWarnings("CdiInjectionPointsInspection") @Inject public PolarisStorageIntegrationProviderImpl( StorageConfiguration storageConfiguration, StsClientProvider stsClientProvider, Clock clock) { this( stsClientProvider, - Optional.ofNullable(storageConfiguration.stsCredentials()), + storageConfiguration.awsSystemCredentials(), storageConfiguration.gcpCredentialsSupplier(clock)); } @@ -115,7 +114,7 @@ public StorageAccessConfig getSubscopedCreds( @Nonnull Set allowedReadLocations, @Nonnull Set allowedWriteLocations, Optional refreshCredentialsEndpoint) { - return StorageAccessConfig.builder().supportsCredentialVending(false).build(); + return StorageAccessConfig.EMPTY; } @Override diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java index f92c45416a..a76fad8f77 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/StorageConfiguration.java @@ -33,7 +33,6 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.StsClientBuilder; @@ -72,28 +71,32 @@ public interface StorageConfiguration extends S3AccessConfig { Optional gcpAccessTokenLifespan(); default Supplier stsClientSupplier() { - return stsClientSupplier(true); - } - - default Supplier stsClientSupplier(boolean withCredentials) { return Suppliers.memoize( () -> { - StsClientBuilder stsClientBuilder = StsClient.builder(); - if (withCredentials) { - stsClientBuilder.credentialsProvider(stsCredentials()); - } - return stsClientBuilder.build(); + StsClientBuilder builder = StsClient.builder(); + awsSystemCredentials().ifPresent(builder::credentialsProvider); + return builder.build(); }); } - default AwsCredentialsProvider stsCredentials() { + /** + * Returns an {@link AwsCredentialsProvider} that provides system-wide AWS credentials. If both + * access key and secret key are present, a static credentials provider is returned. Otherwise, an + * empty optional is returned, implying that the default credentials provider chain should be + * used. + * + *

The returned provider is not meant to be vended directly to clients, but rather used with + * STS, unless credential subscoping is disabled. + */ + default Optional awsSystemCredentials() { if (awsAccessKey().isPresent() && awsSecretKey().isPresent()) { LoggerFactory.getLogger(StorageConfiguration.class) .warn("Using hard-coded AWS credentials - this is not recommended for production"); - return StaticCredentialsProvider.create( - AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get())); + return Optional.of( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(awsAccessKey().get(), awsSecretKey().get()))); } else { - return DefaultCredentialsProvider.builder().build(); + return Optional.empty(); } } diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java new file mode 100644 index 0000000000..675433f1a2 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogAdapter.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.SecurityContext; +import java.util.function.Function; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.service.catalog.CatalogPrefixParser; +import org.apache.polaris.service.catalog.common.CatalogAdapter; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.api.IcebergRestS3SignerApiService; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** A bridge between {@link IcebergRestS3SignerApiService} and {@link CatalogAdapter}. */ +@RequestScoped +public class S3RemoteSigningCatalogAdapter + implements IcebergRestS3SignerApiService, CatalogAdapter { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3RemoteSigningCatalogAdapter.class); + + @Inject PolarisDiagnostics diagnostics; + @Inject CallContext callContext; + @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisAuthorizer polarisAuthorizer; + @Inject CatalogPrefixParser prefixParser; + @Inject S3RequestSigner s3RequestSigner; + @Inject CallContextCatalogFactory callContextCatalogFactory; + @Inject PolarisPrincipal polarisPrincipal; + + /** + * Execute operations on a catalog wrapper and ensure we close the BaseCatalog afterward. This + * will typically ensure the underlying FileIO is closed. + */ + private Response withCatalog( + SecurityContext securityContext, + String prefix, + Function action) { + validatePrincipal(securityContext); + String catalogName = prefixParser.prefixToCatalogName(prefix); + try (S3RemoteSigningCatalogHandler handler = + new S3RemoteSigningCatalogHandler( + diagnostics, + callContext, + resolutionManifestFactory, + callContextCatalogFactory, + polarisPrincipal, + catalogName, + polarisAuthorizer, + s3RequestSigner)) { + return action.apply(handler); + } catch (RuntimeException e) { + LOGGER.debug("RuntimeException while operating on catalog. Propagating to caller.", e); + throw e; + } catch (Exception e) { + LOGGER.error("Error while operating on catalog", e); + throw new RuntimeException(e); + } + } + + @Override + public Response signS3Request( + String prefix, + String namespace, + String table, + PolarisS3SignRequest s3SignRequest, + RealmContext realmContext, + SecurityContext securityContext) { + Namespace ns = decodeNamespace(namespace); + TableIdentifier tableIdentifier = TableIdentifier.of(ns, RESTUtil.decodeString(table)); + return withCatalog( + securityContext, + prefix, + catalog -> { + S3SignResponse response = catalog.signS3Request(s3SignRequest, tableIdentifier); + return Response.status(Response.Status.OK).entity(response).build(); + }); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java new file mode 100644 index 0000000000..4d5d08cd24 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandler.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import java.util.EnumSet; +import java.util.Set; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.auth.PolarisAuthorizableOperation; +import org.apache.polaris.core.auth.PolarisAuthorizer; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.config.FeatureConfiguration; +import org.apache.polaris.core.config.RealmConfig; +import org.apache.polaris.core.context.CallContext; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.core.entity.PolarisEntitySubType; +import org.apache.polaris.core.entity.PolarisEntityType; +import org.apache.polaris.core.entity.table.IcebergTableLikeEntity; +import org.apache.polaris.core.entity.table.TableLikeEntity; +import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper; +import org.apache.polaris.core.persistence.resolver.ResolutionManifestFactory; +import org.apache.polaris.core.storage.LocationRestrictions; +import org.apache.polaris.core.storage.StorageUtil; +import org.apache.polaris.service.catalog.common.CatalogHandler; +import org.apache.polaris.service.catalog.common.CatalogUtils; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignRequest; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3RemoteSigningCatalogHandler extends CatalogHandler implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3RemoteSigningCatalogHandler.class); + + private final CallContextCatalogFactory catalogFactory; + private final S3RequestSigner s3RequestSigner; + + private CatalogEntity catalogEntity; + + public S3RemoteSigningCatalogHandler( + PolarisDiagnostics diagnostics, + CallContext callContext, + ResolutionManifestFactory resolutionManifestFactory, + CallContextCatalogFactory catalogFactory, + PolarisPrincipal polarisPrincipal, + String catalogName, + PolarisAuthorizer authorizer, + S3RequestSigner s3RequestSigner) { + super( + diagnostics, + callContext, + resolutionManifestFactory, + polarisPrincipal, + catalogName, + authorizer, + // external catalogs are not supported for S3 remote signing + null, + null); + this.catalogFactory = catalogFactory; + this.s3RequestSigner = s3RequestSigner; + } + + @Override + protected void initializeCatalog() { + catalogEntity = + CatalogEntity.of(resolutionManifest.getResolvedReferenceCatalogEntity().getRawLeafEntity()); + if (catalogEntity.isExternal()) { + throw new ForbiddenException("Cannot use S3 remote signing with federated catalogs."); + } + // no need to materialize the catalog here, as we only need the catalog entity + } + + public PolarisS3SignResponse signS3Request( + PolarisS3SignRequest s3SignRequest, TableIdentifier tableIdentifier) { + + LOGGER.debug("Requesting s3 signing for {}: {}", tableIdentifier, s3SignRequest); + + PolarisAuthorizableOperation authzOp = + s3SignRequest.write() + ? PolarisAuthorizableOperation.SIGN_S3_WRITE_REQUEST + : PolarisAuthorizableOperation.SIGN_S3_READ_REQUEST; + + authorizeRemoteSigningOrThrow(EnumSet.of(authzOp), tableIdentifier); + + // Must be done after the authorization check, as the auth check creates the catalog entity; + // also, materializing the catalog here could hurt performance. + throwIfRemoteSigningNotEnabled(callContext.getRealmConfig(), catalogEntity); + + validateLocations(s3SignRequest, tableIdentifier); + + PolarisS3SignResponse s3SignResponse = s3RequestSigner.signRequest(s3SignRequest); + LOGGER.debug("S3 signing response: {}", s3SignResponse); + + return s3SignResponse; + } + + private void validateLocations( + PolarisS3SignRequest s3SignRequest, TableIdentifier tableIdentifier) { + + // Will point to the table entity if it exists, otherwise the namespace entity. + PolarisResolvedPathWrapper tableOrNamespace = + CatalogUtils.findResolvedStorageEntity(resolutionManifest, tableIdentifier); + + Set targetLocations = getTargetLocations(s3SignRequest); + + // If the table exists already, validate the target locations against the table's locations; + // otherwise, validate against the namespace's locations using the entity path hierarchy. + if (tableOrNamespace.getRawLeafEntity().getType() == PolarisEntityType.TABLE_LIKE + && tableOrNamespace.getRawLeafEntity().getSubType() == PolarisEntitySubType.ICEBERG_TABLE) { + TableLikeEntity tableEntity = new IcebergTableLikeEntity(tableOrNamespace.getRawLeafEntity()); + Set allowedLocations = + StorageUtil.getLocationsUsedByTable( + tableEntity.getBaseLocation(), tableEntity.getPropertiesAsMap()); + new LocationRestrictions(allowedLocations) + .validate(callContext.getRealmConfig(), tableIdentifier, targetLocations); + } else { + CatalogUtils.validateLocationsForTableLike( + callContext.getRealmConfig(), tableIdentifier, targetLocations, tableOrNamespace); + } + } + + private Set getTargetLocations(PolarisS3SignRequest s3SignRequest) { + // TODO M2: map http URI to s3 URI + return Set.of(); + } + + public static void throwIfRemoteSigningNotEnabled( + RealmConfig realmConfig, CatalogEntity catalogEntity) { + if (catalogEntity.isExternal()) { + throw new ForbiddenException("Remote signing is not enabled for external catalogs."); + } + boolean remoteSigningEnabled = + realmConfig.getConfig(FeatureConfiguration.REMOTE_SIGNING_ENABLED, catalogEntity); + if (!remoteSigningEnabled) { + throw new ForbiddenException( + "Remote signing is not enabled for this catalog. To enable this feature, set the Polaris configuration %s " + + "or the catalog configuration %s.", + FeatureConfiguration.REMOTE_SIGNING_ENABLED.key(), + FeatureConfiguration.REMOTE_SIGNING_ENABLED.catalogConfig()); + } + } + + @Override + public void close() {} +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java new file mode 100644 index 0000000000..63175338be --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSigner.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; + +/** Interface for signing S3 requests. */ +public interface S3RequestSigner { + + /** Signs an S3 request. */ + PolarisS3SignResponse signRequest(S3SignRequest signingRequest); +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java new file mode 100644 index 0000000000..c2ae7db79a --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImpl.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.net.URI; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignResponse; +import org.apache.polaris.service.s3.sign.model.PolarisS3SignResponse; +import org.apache.polaris.service.storage.StorageConfiguration; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.ContentStreamProvider; +import software.amazon.awssdk.http.SdkHttpFullRequest; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.auth.aws.signer.AwsV4HttpSigner; +import software.amazon.awssdk.http.auth.spi.signer.SignRequest; +import software.amazon.awssdk.http.auth.spi.signer.SignedRequest; +import software.amazon.awssdk.services.s3.S3Client; + +@ApplicationScoped +class S3RequestSignerImpl implements S3RequestSigner { + + private final AwsV4HttpSigner signer = AwsV4HttpSigner.create(); + + @Inject StorageConfiguration storageConfiguration; + + @Override + public PolarisS3SignResponse signRequest(S3SignRequest signingRequest) { + + URI uri = signingRequest.uri(); + SdkHttpMethod method = SdkHttpMethod.valueOf(signingRequest.method()); + + SdkHttpFullRequest.Builder requestToSign = + SdkHttpFullRequest.builder() + .uri(uri) + .protocol(uri.getScheme()) + .method(method) + .headers(signingRequest.headers()); + + AwsCredentials credentials = + storageConfiguration + .awsSystemCredentials() + .orElseGet(() -> DefaultCredentialsProvider.builder().build()) + .resolveCredentials(); + + SignRequest.Builder signRequest = + SignRequest.builder(credentials) + .request(requestToSign.build()) + .putProperty(AwsV4HttpSigner.REGION_NAME, signingRequest.region()) + .putProperty(AwsV4HttpSigner.SERVICE_SIGNING_NAME, S3Client.SERVICE_NAME) + .putProperty(AwsV4HttpSigner.DOUBLE_URL_ENCODE, false) + .putProperty(AwsV4HttpSigner.NORMALIZE_PATH, false) + .putProperty(AwsV4HttpSigner.CHUNK_ENCODING_ENABLED, false) + .putProperty(AwsV4HttpSigner.PAYLOAD_SIGNING_ENABLED, false); + + String body = signingRequest.body(); + if (body != null) { + signRequest.payload(ContentStreamProvider.fromUtf8String(body)); + } + + SignedRequest signed = signer.sign(signRequest.build()); + SdkHttpRequest signedRequest = signed.request(); + + return ImmutablePolarisS3SignResponse.builder() + .uri(signedRequest.getUri()) + .headers(signedRequest.headers()) + .build(); + } +} diff --git a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java index cddb671473..8f5b3a1ac8 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/task/TaskFileIOSupplier.java @@ -61,7 +61,7 @@ public FileIO apply(TaskEntity task, TableIdentifier identifier) { PolarisResolvedPathWrapper resolvedPath = new PolarisResolvedPathWrapper(List.of(resolvedTaskEntity)); StorageAccessConfig storageAccessConfig = - accessConfigProvider.getStorageAccessConfig( + accessConfigProvider.getStorageAccessConfigForCredentialsVending( identifier, locations, storageActions, Optional.empty(), resolvedPath); String ioImpl = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java index 0bb4856eb2..0c8bdc997d 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/PolarisAuthzTestBase.java @@ -81,6 +81,7 @@ import org.apache.polaris.core.policy.PredefinedPolicyTypes; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; @@ -91,7 +92,6 @@ import org.apache.polaris.service.catalog.io.StorageAccessConfigProvider; import org.apache.polaris.service.catalog.policy.PolicyCatalog; import org.apache.polaris.service.config.ReservedProperties; -import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.context.catalog.PolarisCallContextCatalogFactory; import org.apache.polaris.service.events.listeners.PolarisEventListener; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; @@ -127,6 +127,7 @@ public Map getConfigOverrides() { .put("polaris.features.\"DROP_WITH_PURGE_ENABLED\"", "true") .put("polaris.behavior-changes.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") .build(); } } @@ -192,7 +193,6 @@ public Map getConfigOverrides() { @Inject protected MetaStoreManagerFactory managerFactory; @Inject protected ResolutionManifestFactory resolutionManifestFactory; - @Inject protected CallContextCatalogFactory callContextCatalogFactory; @Inject protected UserSecretsManagerFactory userSecretsManagerFactory; @Inject protected ServiceIdentityProvider serviceIdentityProvider; @Inject protected PolarisCredentialManager credentialManager; @@ -204,6 +204,7 @@ public Map getConfigOverrides() { @Inject protected StorageCredentialCache storageCredentialCache; @Inject protected ResolverFactory resolverFactory; @Inject protected StorageAccessConfigProvider storageAccessConfigProvider; + @Inject protected PolarisStorageIntegrationProvider storageIntegrationProvider; protected IcebergCatalog baseCatalog; protected PolarisGenericTableCatalog genericTableCatalog; diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java index c2d0e997b6..2c060a0335 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/generic/AbstractPolarisGenericTableCatalogTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Method; import java.util.List; @@ -65,6 +66,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; @@ -107,6 +109,7 @@ public abstract class AbstractPolarisGenericTableCatalogTest { @Inject PolarisDiagnostics diagServices; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject CatalogPrefixParser prefixParser; private PolarisGenericTableCatalog genericTableCatalog; private IcebergCatalog icebergCatalog; @@ -160,7 +163,12 @@ public void before(TestInfo testInfo) { StorageCredentialsVendor storageCredentialsVendor = new StorageCredentialsVendor(metaStoreManager, polarisContext); storageAccessConfigProvider = - new StorageAccessConfigProvider(storageCredentialCache, storageCredentialsVendor); + new StorageAccessConfigProvider( + storageCredentialCache, + storageCredentialsVendor, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java index e9a0b2e777..6d29d4ebb7 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogTest.java @@ -38,6 +38,7 @@ import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.reflect.Method; @@ -135,6 +136,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; @@ -237,6 +239,7 @@ public Map getConfigOverrides() { @Inject ServiceIdentityProvider serviceIdentityProvider; @Inject PolarisDiagnostics diagServices; @Inject PolarisEventListener polarisEventListener; + @Inject CatalogPrefixParser prefixParser; private IcebergCatalog catalog; private String realmName; @@ -294,7 +297,12 @@ public void before(TestInfo testInfo) { StorageCredentialsVendor storageCredentialsVendor = new StorageCredentialsVendor(metaStoreManager, polarisContext); storageAccessConfigProvider = - new StorageAccessConfigProvider(storageCredentialCache, storageCredentialsVendor); + new StorageAccessConfigProvider( + storageCredentialCache, + storageCredentialsVendor, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); EntityCache entityCache = createEntityCache(diagServices, realmConfig, metaStoreManager); resolverFactory = diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java index 7dd45d1808..e4c6cdaa3e 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/AbstractIcebergCatalogViewTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -53,9 +54,11 @@ import org.apache.polaris.core.persistence.resolver.ResolverFactory; import org.apache.polaris.core.secrets.UserSecretsManager; import org.apache.polaris.core.secrets.UserSecretsManagerFactory; +import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; import org.apache.polaris.core.storage.StorageCredentialsVendor; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.Profiles; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; @@ -114,6 +117,8 @@ public Map getConfigOverrides() { @Inject PolarisEventListener polarisEventListener; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject PolarisStorageIntegrationProvider storageIntegrationProvider; + @Inject CatalogPrefixParser prefixParser; private IcebergCatalog catalog; @@ -166,7 +171,12 @@ public void before(TestInfo testInfo) { StorageCredentialsVendor storageCredentialsVendor = new StorageCredentialsVendor(metaStoreManager, polarisContext); storageAccessConfigProvider = - new StorageAccessConfigProvider(storageCredentialCache, storageCredentialsVendor); + new StorageAccessConfigProvider( + storageCredentialCache, + storageCredentialsVendor, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java index 34118d6a71..a722d2b726 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogHandlerAuthzTest.java @@ -24,6 +24,7 @@ import jakarta.enterprise.inject.Instance; import jakarta.inject.Inject; import java.time.Instant; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,6 +70,7 @@ import org.apache.polaris.core.persistence.dao.entity.CreatePrincipalResult; import org.apache.polaris.core.persistence.resolver.PolarisResolutionManifest; import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.catalog.AccessDelegationMode; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; import org.apache.polaris.service.context.catalog.PolarisCallContextCatalogFactory; @@ -642,7 +644,10 @@ public void testCreateTableDirectWithWriteDelegationAllSufficientPrivileges() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableDirectWithWriteDelegation( - NS2, createDirectWithWriteDelegationRequest, Optional.empty()); + NS2, + createDirectWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }, () -> { newWrapper(Set.of(PRINCIPAL_ROLE2)).dropTableWithPurge(newtable); @@ -673,7 +678,10 @@ public void testCreateTableDirectWithWriteDelegationInsufficientPermissions() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableDirectWithWriteDelegation( - NS2, createDirectWithWriteDelegationRequest, Optional.empty()); + NS2, + createDirectWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }); } @@ -747,7 +755,10 @@ public void testCreateTableStagedWithWriteDelegationAllSufficientPrivileges() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableStagedWithWriteDelegation( - NS2, createStagedWithWriteDelegationRequest, Optional.empty()); + NS2, + createStagedWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }, // createTableStagedWithWriteDelegation doesn't actually commit any metadata null, @@ -777,7 +788,10 @@ public void testCreateTableStagedWithWriteDelegationInsufficientPermissions() { () -> { newWrapper(Set.of(PRINCIPAL_ROLE1)) .createTableStagedWithWriteDelegation( - NS2, createStagedWithWriteDelegationRequest, Optional.empty()); + NS2, + createStagedWithWriteDelegationRequest, + Set.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()); }); } @@ -921,7 +935,13 @@ public void testLoadTableWithReadAccessDelegationSufficientPrivileges() { PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty()), + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -937,7 +957,13 @@ public void testLoadTableWithReadAccessDelegationInsufficientPermissions() { PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_LIST, PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty())); + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test @@ -950,7 +976,13 @@ public void testLoadTableWithWriteAccessDelegationSufficientPrivileges() { PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.CATALOG_MANAGE_CONTENT), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty()), + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -966,7 +998,13 @@ public void testLoadTableWithWriteAccessDelegationInsufficientPermissions() { PolarisPrivilege.TABLE_CREATE, PolarisPrivilege.TABLE_LIST, PolarisPrivilege.TABLE_DROP), - () -> newWrapper().loadTableWithAccessDelegation(TABLE_NS1A_2, "all", Optional.empty())); + () -> + newWrapper() + .loadTableWithAccessDelegation( + TABLE_NS1A_2, + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test @@ -979,7 +1017,11 @@ public void testLoadTableWithReadAccessDelegationIfStaleSufficientPrivileges() { () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty()), + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -998,7 +1040,51 @@ public void testLoadTableWithReadAccessDelegationIfStaleInsufficientPermissions( () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty())); + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); + } + + @Test + public void testLoadTableWithReadRemoteSigningIfStaleSufficientPrivileges() { + doTestSufficientPrivileges( + List.of( + PolarisPrivilege.TABLE_READ_DATA, + PolarisPrivilege.TABLE_WRITE_DATA, + PolarisPrivilege.CATALOG_MANAGE_CONTENT), + () -> + newWrapper() + .loadTableWithAccessDelegationIfStale( + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.REMOTE_SIGNING), + Optional.empty()), + null /* cleanupAction */); + } + + @Test + public void testLoadTableWithReadRemoteSigningIfStaleInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of( + PolarisPrivilege.NAMESPACE_FULL_METADATA, + PolarisPrivilege.VIEW_FULL_METADATA, + PolarisPrivilege.TABLE_FULL_METADATA, + PolarisPrivilege.TABLE_READ_PROPERTIES, + PolarisPrivilege.TABLE_WRITE_PROPERTIES, + PolarisPrivilege.TABLE_CREATE, + PolarisPrivilege.TABLE_LIST, + PolarisPrivilege.TABLE_DROP), + () -> + newWrapper() + .loadTableWithAccessDelegationIfStale( + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.REMOTE_SIGNING), + Optional.empty())); } @Test @@ -1014,7 +1100,11 @@ public void testLoadTableWithWriteAccessDelegationIfStaleSufficientPrivileges() () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty()), + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty()), null /* cleanupAction */); } @@ -1033,7 +1123,11 @@ public void testLoadTableWithWriteAccessDelegationIfStaleInsufficientPermissions () -> newWrapper() .loadTableWithAccessDelegationIfStale( - TABLE_NS1A_2, IfNoneMatch.fromHeader("W/\"0:0\""), "all", Optional.empty())); + TABLE_NS1A_2, + IfNoneMatch.fromHeader("W/\"0:0\""), + "all", + EnumSet.of(AccessDelegationMode.VENDED_CREDENTIALS), + Optional.empty())); } @Test diff --git a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java index eba010e11a..3b54bd2b60 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/catalog/policy/AbstractPolicyCatalogTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap; import io.quarkus.test.junit.QuarkusMock; import jakarta.inject.Inject; +import jakarta.ws.rs.core.UriInfo; import java.io.IOException; import java.lang.reflect.Method; import java.util.Arrays; @@ -77,6 +78,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.cache.StorageCredentialCache; import org.apache.polaris.service.admin.PolarisAdminService; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView; import org.apache.polaris.service.catalog.iceberg.IcebergCatalog; import org.apache.polaris.service.catalog.io.DefaultFileIOFactory; @@ -133,6 +135,7 @@ public abstract class AbstractPolicyCatalogTest { @Inject PolarisDiagnostics diagServices; @Inject ResolverFactory resolverFactory; @Inject ResolutionManifestFactory resolutionManifestFactory; + @Inject CatalogPrefixParser prefixParser; private PolicyCatalog policyCatalog; private IcebergCatalog icebergCatalog; @@ -181,7 +184,12 @@ public void before(TestInfo testInfo) { StorageCredentialsVendor storageCredentialsVendor = new StorageCredentialsVendor(metaStoreManager, polarisContext); storageAccessConfigProvider = - new StorageAccessConfigProvider(storageCredentialCache, storageCredentialsVendor); + new StorageAccessConfigProvider( + storageCredentialCache, + storageCredentialsVendor, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); PrincipalEntity rootPrincipal = metaStoreManager.findRootPrincipal(polarisContext).orElseThrow(); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java new file mode 100644 index 0000000000..57ab3baac3 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialTest.java @@ -0,0 +1,453 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX; +import static org.apache.iceberg.aws.AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.ACCESS_KEY_ID; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.ENDPOINT; +import static org.apache.iceberg.aws.s3.S3FileIOProperties.SECRET_ACCESS_KEY; +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_KEY_ID; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY; +import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.PositionOutputStream; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.types.Types; +import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.service.catalog.AccessDelegationMode; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; + +/** + * These tests complement {@link PolarisRestCatalogMinIOIT} to validate client-side access to MinIO + * storage via {@code FileIO} instances configured from catalog's {@code loadTable} responses with + * some S3-specific options. + */ +@QuarkusTest +@TestProfile(RestCatalogMinIOSpecialTest.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class RestCatalogMinIOSpecialTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + private static String adminToken; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.IntegerType.get(), "doc"), + optional(2, "data", Types.StringType.get())); + + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + private static URI storageBase; + private static String endpoint; + private static S3Client s3Client; + + private CatalogApi catalogApi; + private String principalRoleName; + private PrincipalWithCredentials principalCredentials; + private String catalogName; + + @BeforeAll + static void setup( + PolarisApiEndpoints apiEndpoints, + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess, + ClientCredentials credentials) { + s3Client = minioAccess.s3Client(); + endpoints = apiEndpoints; + client = polarisClient(endpoints); + adminToken = client.obtainToken(credentials); + managementApi = client.managementApi(adminToken); + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @AfterAll + static void close() throws Exception { + client.close(); + } + + @BeforeEach + public void before(TestInfo testInfo) { + String principalName = client.newEntityName("test-user"); + principalRoleName = client.newEntityName("test-admin"); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + + String principalToken = client.obtainToken(principalCredentials); + catalogApi = client.catalogApi(principalToken); + + catalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName()); + } + + private RESTCatalog createCatalog( + Optional endpoint, + Optional stsEndpoint, + boolean pathStyleAccess, + Optional delegationMode, + boolean stsEnabled) { + return createCatalog( + endpoint, stsEndpoint, pathStyleAccess, Optional.empty(), delegationMode, stsEnabled); + } + + private RESTCatalog createCatalog( + Optional endpoint, + Optional stsEndpoint, + boolean pathStyleAccess, + Optional endpointInternal, + Optional delegationMode, + boolean stsEnabled) { + AwsStorageConfigInfo.Builder storageConfig = + AwsStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setPathStyleAccess(pathStyleAccess) + .setStsUnavailable(!stsEnabled) + .setAllowedLocations(List.of(storageBase.toString())); + + endpoint.ifPresent(storageConfig::setEndpoint); + stsEndpoint.ifPresent(storageConfig::setStsEndpoint); + endpointInternal.ifPresent(storageConfig::setEndpointInternal); + + CatalogProperties.Builder catalogProps = + CatalogProperties.builder(storageBase.toASCIIString() + "/" + catalogName); + if (!stsEnabled) { + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY); + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setStorageConfigInfo(storageConfig.build()) + .setProperties(catalogProps.build()) + .build(); + + managementApi.createCatalog(principalRoleName, catalog); + + String authToken = client.obtainToken(principalCredentials); + RESTCatalog restCatalog = new RESTCatalog(); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put( + org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) + .put(OAuth2Properties.TOKEN, authToken) + .put("warehouse", catalogName) + .putAll(endpoints.extraHeaders("header.")); + + delegationMode.ifPresent( + dm -> propertiesBuilder.put("header.X-Iceberg-Access-Delegation", dm.protocolValue())); + + if (delegationMode.isEmpty()) { + // Use local credentials on the client side + propertiesBuilder.put("s3.access-key-id", MINIO_ACCESS_KEY); + propertiesBuilder.put("s3.secret-access-key", MINIO_SECRET_KEY); + } + + restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + return restCatalog; + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminToken); + } + + @ParameterizedTest + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + public void testCreateTable(boolean pathStyle, boolean stsEnabled) throws IOException { + LoadTableResponse response = doTestCreateTable(pathStyle, Optional.empty(), stsEnabled); + assertThat(response.config()).doesNotContainKey(SECRET_ACCESS_KEY); + assertThat(response.config()).doesNotContainKey(ACCESS_KEY_ID); + assertThat(response.config()).doesNotContainKey(REFRESH_CREDENTIALS_ENDPOINT); + assertThat(response.credentials()).isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testCreateTableVendedCredentials(boolean pathStyle) throws IOException { + LoadTableResponse response = + doTestCreateTable(pathStyle, Optional.of(VENDED_CREDENTIALS), true); + assertThat(response.config()) + .containsEntry( + REFRESH_CREDENTIALS_ENDPOINT, + "v1/" + catalogName + "/namespaces/test-ns/tables/t1/credentials"); + assertThat(response.credentials()).hasSize(1); + } + + private LoadTableResponse doTestCreateTable( + boolean pathStyle, Optional dm, boolean stsEnabled) throws IOException { + try (RESTCatalog restCatalog = + createCatalog(Optional.of(endpoint), Optional.empty(), pathStyle, dm, stsEnabled)) { + LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog, dm); + if (pathStyle) { + assertThat(loadTableResponse.config()) + .containsEntry("s3.path-style-access", Boolean.TRUE.toString()); + } + return loadTableResponse; + } + } + + @Test + public void testInternalEndpoints() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of("http://s3.example.com"), + Optional.of(endpoint), + false, + Optional.of(endpoint), + Optional.empty(), + true)) { + StorageConfigInfo storageConfig = + managementApi.getCatalog(catalogName).getStorageConfigInfo(); + assertThat((AwsStorageConfigInfo) storageConfig) + .extracting( + AwsStorageConfigInfo::getEndpoint, + AwsStorageConfigInfo::getStsEndpoint, + AwsStorageConfigInfo::getEndpointInternal, + AwsStorageConfigInfo::getPathStyleAccess) + .containsExactly("http://s3.example.com", endpoint, endpoint, false); + LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog, Optional.empty()); + assertThat(loadTableResponse.config()).containsEntry(ENDPOINT, "http://s3.example.com"); + } + } + + @Test + public void testCreateTableFailureWithCredentialVendingWithoutSts() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of("http://sts.example.com"), // not called + false, + Optional.of(VENDED_CREDENTIALS), + false)) { + StorageConfigInfo storageConfig = + managementApi.getCatalog(catalogName).getStorageConfigInfo(); + assertThat((AwsStorageConfigInfo) storageConfig) + .extracting( + AwsStorageConfigInfo::getEndpoint, + AwsStorageConfigInfo::getStsEndpoint, + AwsStorageConfigInfo::getEndpointInternal, + AwsStorageConfigInfo::getPathStyleAccess, + AwsStorageConfigInfo::getStsUnavailable) + .containsExactly(endpoint, "http://sts.example.com", null, false, true); + + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t2"); + // Credential vending is not supported without STS + assertThatThrownBy(() -> restCatalog.createTable(id, SCHEMA)) + .hasMessageContaining("but no credentials are available") + .hasMessageContaining(id.toString()); + } + } + + @Test + public void testLoadTableFailureWithCredentialVendingWithoutSts() throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of("http://sts.example.com"), // not called + false, + Optional.empty(), + false)) { + + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t3"); + restCatalog.createTable(id, SCHEMA); + + // Credential vending is not supported without STS + assertThatThrownBy( + () -> + catalogApi.loadTable( + catalogName, + id, + "ALL", + Map.of("X-Iceberg-Access-Delegation", VENDED_CREDENTIALS.protocolValue()))) + .hasMessageContaining("but no credentials are available") + .hasMessageContaining(id.toString()); + } + } + + public LoadTableResponse doTestCreateTable( + RESTCatalog restCatalog, Optional dm) { + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t1"); + Table table = restCatalog.createTable(id, SCHEMA); + assertThat(table).isNotNull(); + assertThat(restCatalog.tableExists(id)).isTrue(); + + TableOperations ops = ((HasTableOperations) table).operations(); + URI location = URI.create(ops.current().metadataFileLocation()); + + GetObjectResponse response = + s3Client + .getObject( + GetObjectRequest.builder() + .bucket(location.getAuthority()) + .key(location.getPath().substring(1)) // drop leading slash + .build()) + .response(); + assertThat(response.contentLength()).isGreaterThan(0); + + LoadTableResponse loadTableResponse = + catalogApi.loadTable( + catalogName, + id, + "ALL", + dm.map(v -> Map.of("X-Iceberg-Access-Delegation", v.protocolValue())).orElse(Map.of())); + + assertThat(loadTableResponse.config()).containsKey(ENDPOINT); + + restCatalog.dropTable(id); + assertThat(restCatalog.tableExists(id)).isFalse(); + return loadTableResponse; + } + + @ParameterizedTest + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + @CsvSource("true, true, VENDED_CREDENTIALS") + @CsvSource("false, true, VENDED_CREDENTIALS") + public void testAppendFiles( + boolean pathStyle, boolean stsEnabled, AccessDelegationMode delegationMode) + throws IOException { + try (RESTCatalog restCatalog = + createCatalog( + Optional.of(endpoint), + Optional.of(endpoint), + pathStyle, + Optional.ofNullable(delegationMode), + stsEnabled)) { + catalogApi.createNamespace(catalogName, "test-ns"); + TableIdentifier id = TableIdentifier.of("test-ns", "t1"); + Table table = restCatalog.createTable(id, SCHEMA); + assertThat(table).isNotNull(); + + @SuppressWarnings("resource") + FileIO io = table.io(); + + URI loc = + URI.create( + table + .locationProvider() + .newDataLocation( + String.format( + "test-file-%s-%s-%s.txt", pathStyle, delegationMode, stsEnabled))); + OutputFile f1 = io.newOutputFile(loc.toString()); + try (PositionOutputStream os = f1.create()) { + os.write("Hello World".getBytes(UTF_8)); + } + + DataFile df = + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(f1.location()) + .withFormat(FileFormat.PARQUET) // bogus value + .withFileSizeInBytes(4) + .withRecordCount(1) + .build(); + + table.newAppend().appendFile(df).commit(); + + try (InputStream is = + s3Client.getObject( + GetObjectRequest.builder() + .bucket(loc.getAuthority()) + .key(loc.getPath().substring(1)) // drop leading slash + .build())) { + assertThat(new String(is.readAllBytes(), UTF_8)).isEqualTo("Hello World"); + } + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java new file mode 100644 index 0000000000..9c986bfae5 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/S3RemoteSigningMinIOIntegrationTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.polaris.core.storage.StorageAccessProperty; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.apache.polaris.service.it.test.PolarisS3RemoteSigningIntegrationTest; +import org.apache.polaris.test.minio.Minio; +import org.apache.polaris.test.minio.MinioAccess; +import org.apache.polaris.test.minio.MinioExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; + +@QuarkusTest +@TestProfile(S3RemoteSigningMinIOIntegrationTest.Profile.class) +@ExtendWith(MinioExtension.class) +@ExtendWith(PolarisIntegrationTestExtension.class) +public class S3RemoteSigningMinIOIntegrationTest extends PolarisS3RemoteSigningIntegrationTest { + + private static final String BUCKET_URI_PREFIX = "/minio-test"; + private static final String MINIO_ACCESS_KEY = "test-ak-123"; + private static final String MINIO_SECRET_KEY = "test-sk-123"; + + public static class Profile implements QuarkusTestProfile { + + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .put("polaris.storage.aws.access-key", MINIO_ACCESS_KEY) + .put("polaris.storage.aws.secret-key", MINIO_SECRET_KEY) + .put("polaris.readiness.ignore-severe-issues", "true") + .put("polaris.features.\"REMOTE_SIGNING_ENABLED\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"S3\"]") + .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") + .build(); + } + } + + private static URI storageBase; + private static String endpoint; + + @BeforeAll + static void setup( + @Minio(accessKey = MINIO_ACCESS_KEY, secretKey = MINIO_SECRET_KEY) MinioAccess minioAccess) { + storageBase = minioAccess.s3BucketUri(BUCKET_URI_PREFIX); + endpoint = minioAccess.s3endpoint(); + } + + @Override + protected List allowedLocations() { + return List.of( + storageBase.resolve(BUCKET_URI_PREFIX + "/allowed-location1").toString(), + storageBase.resolve(BUCKET_URI_PREFIX + "/allowed-location2").toString()); + } + + @Override + protected Optional endpoint() { + return Optional.of(endpoint); + } + + @Override + protected ImmutableMap.Builder clientFileIOProperties() { + return super.clientFileIOProperties() + // Grant direct access to the MinIO bucket; this FileIO instance does not + // use access delegation. + .put(StorageAccessProperty.AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY) + .put(StorageAccessProperty.AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java index 916b1912a1..91c9db8643 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/StorageConfigurationTest.java @@ -192,7 +192,7 @@ public void testSingletonStsClientWithStaticCredentials() { staticMock.when(StsClient::builder).thenReturn(mockBuilder); StorageConfiguration config = configWithAwsCredentialsAndGcpToken(); - Supplier supplier = config.stsClientSupplier(true); + Supplier supplier = config.stsClientSupplier(); StsClient client1 = supplier.get(); StsClient client2 = supplier.get(); @@ -209,7 +209,7 @@ public void testSingletonStsClientWithStaticCredentials() { @Test public void testStaticStsCredentials() { StorageConfiguration config = configWithAwsCredentialsAndGcpToken(); - AwsCredentialsProvider credentialsProvider = config.stsCredentials(); + AwsCredentialsProvider credentialsProvider = config.awsSystemCredentials().orElseThrow(); assertThat(credentialsProvider).isInstanceOf(StaticCredentialsProvider.class); assertThat(credentialsProvider.resolveCredentials().accessKeyId()).isEqualTo(TEST_ACCESS_KEY); assertThat(credentialsProvider.resolveCredentials().secretAccessKey()) diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java new file mode 100644 index 0000000000..c797195740 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RemoteSigningCatalogHandlerAuthzTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.storage.s3.sign; + +import static org.mockito.ArgumentMatchers.any; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import java.net.URI; +import java.util.List; +import java.util.Set; +import org.apache.polaris.core.auth.PolarisPrincipal; +import org.apache.polaris.core.entity.PolarisPrivilege; +import org.apache.polaris.service.admin.PolarisAuthzTestBase; +import org.apache.polaris.service.context.catalog.CallContextCatalogFactory; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignRequest; +import org.apache.polaris.service.s3.sign.model.ImmutablePolarisS3SignResponse; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +@QuarkusTest +@TestProfile(PolarisAuthzTestBase.Profile.class) +@SuppressWarnings("resource") +public class S3RemoteSigningCatalogHandlerAuthzTest extends PolarisAuthzTestBase { + + private static final ImmutablePolarisS3SignRequest READ_REQUEST = + ImmutablePolarisS3SignRequest.builder() + .method("GET") + .uri(URI.create("https://example-bucket.s3.amazonaws.com/some-object")) + .region("us-west-2") + .build(); + + private static final ImmutablePolarisS3SignRequest WRITE_REQUEST = + ImmutablePolarisS3SignRequest.builder() + .method("PUT") + .uri(URI.create("https://example-bucket.s3.amazonaws.com/some-object")) + .region("us-west-2") + .build(); + + @Test + public void testReadRequestInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of(PolarisPrivilege.TABLE_REMOTE_SIGN, PolarisPrivilege.TABLE_READ_DATA), + () -> newHandler().signS3Request(READ_REQUEST, TABLE_NS1_1)); + } + + @Test + public void testWriteRequestInsufficientPermissions() { + doTestInsufficientPrivileges( + List.of(PolarisPrivilege.TABLE_REMOTE_SIGN, PolarisPrivilege.TABLE_WRITE_DATA), + () -> newHandler().signS3Request(WRITE_REQUEST, TABLE_NS1_1)); + } + + @Test + public void testReadRequestSufficientPermissions() { + doTestSufficientPrivilegeSets( + List.of(Set.of(PolarisPrivilege.TABLE_READ_DATA, PolarisPrivilege.TABLE_REMOTE_SIGN)), + () -> newHandler().signS3Request(READ_REQUEST, TABLE_NS1_1), + () -> {}, + PRINCIPAL_NAME, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } + + @Test + public void testWriteRequestSufficientPermissions() { + doTestSufficientPrivilegeSets( + List.of(Set.of(PolarisPrivilege.TABLE_WRITE_DATA, PolarisPrivilege.TABLE_REMOTE_SIGN)), + () -> newHandler().signS3Request(WRITE_REQUEST, TABLE_NS1_1), + () -> {}, + PRINCIPAL_NAME, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } + + private S3RemoteSigningCatalogHandler newHandler() { + PolarisPrincipal principal = PolarisPrincipal.of(principalEntity, Set.of()); + S3RequestSigner s3signer = Mockito.mock(S3RequestSigner.class); + Mockito.when(s3signer.signRequest(any())) + .thenReturn(ImmutablePolarisS3SignResponse.builder().uri(URI.create("irrelevant")).build()); + CallContextCatalogFactory callContextCatalogFactory = + Mockito.mock(CallContextCatalogFactory.class); + Mockito.when(callContextCatalogFactory.createCallContextCatalog(any())).thenReturn(baseCatalog); + return new S3RemoteSigningCatalogHandler( + diagServices, + callContext, + resolutionManifestFactory, + callContextCatalogFactory, + principal, + CATALOG_NAME, + polarisAuthorizer, + s3signer); + } + + private void doTestInsufficientPrivileges( + List insufficientPrivileges, Runnable action) { + doTestInsufficientPrivileges( + insufficientPrivileges, + PRINCIPAL_NAME, + action, + (privilege) -> + adminService.grantPrivilegeOnCatalogToRole(CATALOG_NAME, CATALOG_ROLE1, privilege), + (privilege) -> + adminService.revokePrivilegeOnCatalogFromRole(CATALOG_NAME, CATALOG_ROLE1, privilege)); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java new file mode 100644 index 0000000000..1db8a8b4c1 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/storage/s3/sign/S3RequestSignerImplTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.service.storage.s3.sign; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.iceberg.aws.s3.signer.ImmutableS3SignRequest; +import org.apache.iceberg.aws.s3.signer.S3SignRequest; +import org.apache.iceberg.aws.s3.signer.S3SignResponse; +import org.apache.polaris.service.storage.StorageConfiguration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +@ExtendWith(MockitoExtension.class) +class S3RequestSignerImplTest { + + @Mock private StorageConfiguration storageConfiguration; + @Mock private AwsCredentialsProvider awsCredentialsProvider; + + @InjectMocks private S3RequestSignerImpl s3RequestSigner; + + private static final String TEST_ACCESS_KEY = "test-access-key"; + private static final String TEST_SECRET_KEY = "test-secret-key"; + private static final String TEST_REGION = "us-west-2"; + private static final String TEST_HOST = "test-bucket.s3.us-west-2.amazonaws.com"; + private static final URI TEST_URI = URI.create("https://" + TEST_HOST + "/test-path"); + + @BeforeEach + void setUp() { + AwsCredentials credentials = AwsBasicCredentials.create(TEST_ACCESS_KEY, TEST_SECRET_KEY); + when(storageConfiguration.awsSystemCredentials()) + .thenReturn(Optional.of(awsCredentialsProvider)); + when(awsCredentialsProvider.resolveCredentials()).thenReturn(credentials); + } + + @Test + void testGET() { + // Given + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testPUT() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("PUT") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testPOST() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("POST") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testDELETE() { + // Given + String requestBody = "{\"test\": \"data\"}"; + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("DELETE") + .uri(TEST_URI) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .body(requestBody) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + } + + @Test + void testQueryParameters() { + // Given + URI uriWithQuery = URI.create(TEST_URI + "?prefix=test&max-keys=100"); + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(uriWithQuery) + .headers(Map.of("Host", List.of(TEST_HOST))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri().getHost()).isEqualTo(uriWithQuery.getHost()); + assertThat(response.uri().getPath()).isEqualTo(uriWithQuery.getPath()); + assertThat(response.uri().getQuery()).contains("prefix=test"); + assertThat(response.uri().getQuery()).contains("max-keys=100"); + assertHeaders(response); + } + + @Test + void testHeaders() { + // Given + S3SignRequest request = + ImmutableS3SignRequest.builder() + .region(TEST_REGION) + .method("GET") + .uri(TEST_URI) + .headers( + Map.of( + "Host", List.of(TEST_HOST), + "x-amz-content-sha256", List.of("UNSIGNED-PAYLOAD"), + "Content-Type", List.of("application/json"), + "User-Agent", List.of("test-client/1.0"))) + .properties(Map.of()) + .build(); + + // When + S3SignResponse response = s3RequestSigner.signRequest(request); + + // Then + assertThat(response).isNotNull(); + assertThat(response.uri()).isEqualTo(TEST_URI); + assertHeaders(response); + assertThat(response.headers().get("x-amz-content-sha256").getFirst()) + .isEqualTo("UNSIGNED-PAYLOAD"); + assertThat(response.headers().get("Content-Type").getFirst()).isEqualTo("application/json"); + assertThat(response.headers().get("User-Agent").getFirst()).isEqualTo("test-client/1.0"); + } + + private static void assertHeaders(S3SignResponse response) { + assertThat(response.headers()).isNotEmpty(); + assertThat(response.headers()).containsKey("X-Amz-Date"); + assertThat(response.headers()).containsKey("x-amz-content-sha256"); + assertThat(response.headers().get("x-amz-content-sha256").getFirst()) + .isEqualTo("UNSIGNED-PAYLOAD"); // Fixed for S3 + assertThat(response.headers()).containsKey("Authorization"); + assertThat(response.headers().get("Authorization")).hasSize(1); + String authHeader = response.headers().get("Authorization").getFirst(); + assertThat(authHeader).startsWith("AWS4-HMAC-SHA256 Credential=" + TEST_ACCESS_KEY); + assertThat(authHeader).contains("SignedHeaders="); + assertThat(authHeader).contains("Signature="); + } +} diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java index 4c910ffbd6..45d94b8a2e 100644 --- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -27,6 +27,7 @@ import jakarta.annotation.Nullable; import jakarta.enterprise.inject.Instance; import jakarta.ws.rs.core.SecurityContext; +import jakarta.ws.rs.core.UriInfo; import java.security.Principal; import java.time.Clock; import java.time.Instant; @@ -66,6 +67,7 @@ import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; +import org.apache.polaris.service.catalog.CatalogPrefixParser; import org.apache.polaris.service.catalog.DefaultCatalogPrefixParser; import org.apache.polaris.service.catalog.api.IcebergRestCatalogApi; import org.apache.polaris.service.catalog.api.IcebergRestConfigurationApi; @@ -276,8 +278,15 @@ public String getAuthenticationScheme() { StorageCredentialsVendor storageCredentialsVendor = new StorageCredentialsVendor(metaStoreManager, callContext); + CatalogPrefixParser prefixParser = new DefaultCatalogPrefixParser(); StorageAccessConfigProvider storageAccessConfigProvider = - new StorageAccessConfigProvider(storageCredentialCache, storageCredentialsVendor); + new StorageAccessConfigProvider( + storageCredentialCache, + storageCredentialsVendor, + storageIntegrationProvider, + prefixParser, + Mockito.mock(UriInfo.class)); + FileIOFactory fileIOFactory = fileIOFactorySupplier.get(); TaskExecutor taskExecutor = Mockito.mock(TaskExecutor.class); diff --git a/site/content/in-dev/unreleased/managing-security/access-control.md b/site/content/in-dev/unreleased/managing-security/access-control.md index b8a1b697ca..6ce1cf1757 100644 --- a/site/content/in-dev/unreleased/managing-security/access-control.md +++ b/site/content/in-dev/unreleased/managing-security/access-control.md @@ -102,18 +102,19 @@ To grant the full set of privileges (drop, list, read, write, etc.) on an object ### Table privileges -| Privilege | Description | -| --------- | ----------- | -| TABLE_CREATE | Enables registering a table with the catalog. | -| TABLE_DROP | Enables dropping a table from the catalog. | -| TABLE_LIST | Enables listing any table in the catalog. | -| TABLE_READ_PROPERTIES | Enables reading properties of the table. | -| TABLE_WRITE_PROPERTIES | Enables configuring properties for the table. | -| TABLE_READ_DATA | Enables reading data from the table by receiving short-lived read-only storage credentials from the catalog. | -| TABLE_WRITE_DATA | Enables writing data to the table by receiving short-lived read+write storage credentials from the catalog. | -| TABLE_FULL_METADATA | Grants all table privileges, except TABLE_READ_DATA and TABLE_WRITE_DATA, which need to be granted individually. | -| TABLE_ATTACH_POLICY | Enables attaching policy to a table. | -| TABLE_DETACH_POLICY | Enables detaching policy from a table. | +| Privilege | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------| +| TABLE_CREATE | Enables registering a table with the catalog. | +| TABLE_DROP | Enables dropping a table from the catalog. | +| TABLE_LIST | Enables listing any table in the catalog. | +| TABLE_READ_PROPERTIES | Enables reading properties of the table. | +| TABLE_WRITE_PROPERTIES | Enables configuring properties for the table. | +| TABLE_READ_DATA | Enables reading data from the table by receiving short-lived read-only storage credentials from the catalog. | +| TABLE_WRITE_DATA | Enables writing data to the table by receiving short-lived read+write storage credentials from the catalog. | +| TABLE_FULL_METADATA | Grants all table privileges, except TABLE_READ_DATA and TABLE_WRITE_DATA, which need to be granted individually. | +| TABLE_ATTACH_POLICY | Enables attaching policy to a table. | +| TABLE_DETACH_POLICY | Enables detaching policy from a table. | +| TABLE_REMOTE_SIGN | Enables remote signing for a table. TABLE_READ_DATA and TABLE_WRITE_DATA must also be granted individually. | ### View privileges diff --git a/spec/README.md b/spec/README.md index 8d2c0d0f03..567346a165 100644 --- a/spec/README.md +++ b/spec/README.md @@ -17,14 +17,41 @@ under the License. --> -# Polaris API Specifications - -Polaris provides two sets of OpenAPI specifications: -- `polaris-management-service.yml` - Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals -- `polaris-catalog-service.yaml` - Defines the specification for the Polaris Catalog API, which encompasses both the Iceberg REST Catalog API - and Polaris-native API. - - `polaris-catalog-apis` - Contains the specification for Polaris-specific Catalog APIs - - `iceberg-rest-catalog-open-api.yaml` - Contains the specification for Iceberg Rest Catalog API +# Apache Polaris API Specifications + +Apache Polaris provides the following OpenAPI specifications: + +- [polaris-management-service.yml](polaris-management-service.yml) - Defines the management APIs for using Apache + Polaris to create and manage Apache Iceberg catalogs and their principals + +- [polaris-catalog-service.yaml](polaris-catalog-service.yaml) - Defines the specification for the Apache Polaris + Catalog API, which encompasses both the Apache Iceberg REST Catalog API and Apache Polaris-native APIs: + + - [iceberg-rest-catalog-open-api.yaml](iceberg-rest-catalog-open-api.yaml) - Contains the specification for + Apache Iceberg Rest Catalog API. + + - [polaris-catalog-apis](polaris-catalog-apis) - This folder contains the specifications for Apache + Polaris-specific Catalog APIs: + + - [generic-tables-api.yaml](polaris-catalog-apis/generic-tables-api.yaml) - Contains the specification for + the Generic Tables API + + - [notifications-api.yaml](polaris-catalog-apis/notifications-api.yaml) - Contains the specification for + the Notifications API + + - [oauth-tokens-api.yaml](polaris-catalog-apis/oauth-tokens-api.yaml) - Contains the specification for the + internal OAuth Token endpoint, extracted from the Apache Iceberg REST Catalog API. + + - [policy-apis.yaml](polaris-catalog-apis/policy-apis.yaml) - Contains the specification for the Policy APIs. + +- [s3-sign](s3-sign) - This folder contains the specifications for S3 remote signing: + + - [iceberg-s3-signer-open-api.yaml](s3-sign/iceberg-s3-signer-open-api.yaml) - Contains the specification of the + original Apache Iceberg S3 Signer API. Only the type definitions from this file are used; the actual API + endpoints are not used as Polaris uses custom endpoints (see next file). + + - [polaris-s3-sign-service.yaml](s3-sign/polaris-s3-sign-service.yaml) - Contains the Apache Polaris-specific + S3 Signer API. ## Generated Specification Files The specification files in the generated folder are automatically created using OpenAPI bundling tools such as diff --git a/spec/s3-sign/iceberg-s3-signer-open-api.yaml b/spec/s3-sign/iceberg-s3-signer-open-api.yaml new file mode 100644 index 0000000000..92c17a8e6a --- /dev/null +++ b/spec/s3-sign/iceberg-s3-signer-open-api.yaml @@ -0,0 +1,154 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# CODE_COPIED_TO_POLARIS + +# Apache Iceberg Version: 1.10.0 + +--- +openapi: 3.0.3 +info: + title: Apache Iceberg S3 Signer API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the S3 Signer API. +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be prepended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + basePath: + description: Optional prefix to be appended to all routes + default: "" + +paths: + + /v1/aws/s3/sign: + + post: + tags: + - S3 Signer API + summary: Remotely signs S3 requests + operationId: signS3Request + requestBody: + description: The request containing the headers to be signed + content: + application/json: + schema: + $ref: '#/components/schemas/S3SignRequest' + required: true + responses: + 200: + $ref: '#/components/responses/S3SignResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 419: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + + ############################## + # Application Schema Objects # + ############################## +components: + schemas: + + S3Headers: + type: object + additionalProperties: + type: array + items: + type: string + + S3SignRequest: + required: + - region + - uri + - method + - headers + properties: + region: + type: string + uri: + type: string + method: + type: string + enum: ["PUT", "GET", "HEAD", "POST", "DELETE", "PATCH", "OPTIONS"] + headers: + $ref: '#/components/schemas/S3Headers' + properties: + type: object + additionalProperties: + type: string + body: + type: string + description: Optional body of the S3 request to send to the signing API. This should only be populated + for S3 requests which do not have the relevant data in the URI itself (e.g. DeleteObjects requests) + + + ############################# + # Reusable Response Objects # + ############################# + responses: + + S3SignResponse: + description: The response containing signed & unsigned headers. The server will also send + a Cache-Control header, indicating whether the response can be cached (Cache-Control = ["private"]) + or not (Cache-Control = ["no-cache"]). + content: + application/json: + schema: + type: object + required: + - uri + - headers + properties: + uri: + type: string + headers: + $ref: '#/components/schemas/S3Headers' diff --git a/spec/s3-sign/polaris-s3-sign-service.yaml b/spec/s3-sign/polaris-s3-sign-service.yaml new file mode 100644 index 0000000000..7e37b44117 --- /dev/null +++ b/spec/s3-sign/polaris-s3-sign-service.yaml @@ -0,0 +1,108 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Polaris-specific S3 Signer API that includes path parameters required for Polaris, such as +# the prefix, namespace, and table. + +openapi: 3.0.3 +info: + title: Apache Polaris S3 Signer API + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 0.0.1 + description: + Defines the specification for the S3 Signer API integrated with Polaris catalog structure. + +# The server configuration is sourced from iceberg-s3-signer-open-api.yaml +servers: + - url: "{scheme}://{host}/{basePath}" + description: Server URL when the port can be inferred from the scheme + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + basePath: + description: Optional prefix to be appended to all routes + default: "" + - url: "{scheme}://{host}:{port}/{basePath}" + description: Generic base server URL, with all parts configurable + variables: + scheme: + description: The scheme of the URI, either http or https. + default: https + host: + description: The host address for the specified server + default: localhost + port: + description: The port used when addressing the host + default: "443" + +# All routes are currently configured using an Authorization header. +security: + - OAuth2: [] + +paths: + + /v1/{prefix}/namespaces/{namespace}/tables/{table}: + parameters: + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/prefix' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/namespace' + - $ref: '../iceberg-rest-catalog-open-api.yaml#/components/parameters/table' + + post: + tags: + - S3 Signer API + summary: Remotely signs S3 requests + operationId: signS3Request + requestBody: + description: The request containing the headers to be signed + content: + application/json: + schema: + $ref: './iceberg-s3-signer-open-api.yaml#/components/schemas/S3SignRequest' + required: true + responses: + 200: + $ref: './iceberg-s3-signer-open-api.yaml#/components/responses/S3SignResponse' + 400: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/BadRequestErrorResponse' + 401: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/UnauthorizedResponse' + 403: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ForbiddenResponse' + 419: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '../iceberg-rest-catalog-open-api.yaml#/components/responses/ServerErrorResponse' + +components: + securitySchemes: + OAuth2: + type: oauth2 + description: Uses OAuth 2 with client credentials flow + flows: + clientCredentials: + tokenUrl: "{scheme}://{host}/api/v1/oauth/tokens" + scopes: {} \ No newline at end of file