diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts index 6a2f587e50..47f6ab80f9 100644 --- a/integration-tests/build.gradle.kts +++ b/integration-tests/build.gradle.kts @@ -62,6 +62,7 @@ dependencies { implementation(platform(libs.junit.bom)) implementation("org.junit.jupiter:junit-jupiter") + implementation("org.junit.jupiter:junit-jupiter-api") compileOnly("org.junit.jupiter:junit-jupiter-engine") implementation(libs.assertj.core) implementation(libs.mockito.core) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java new file mode 100644 index 0000000000..7e2a8e41aa --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import com.google.common.annotations.VisibleForTesting; +import java.net.URI; +import java.nio.file.Path; +import java.util.function.Function; + +public final class IntegrationTestsHelper { + + /** + * The environment variable that can be used to override the temporary directory used by the + * integration tests. + */ + public static final String INTEGRATION_TEST_TEMP_DIR_ENV_VAR = "INTEGRATION_TEST_TEMP_DIR"; + + private IntegrationTestsHelper() {} + + /** + * Get the temporary directory to use for integration tests. + * + *

If the environment variable {@link #INTEGRATION_TEST_TEMP_DIR_ENV_VAR} is set, it will be + * used as the temporary directory. Otherwise, the default local temporary directory will be used. + * + *

The environment variable should be a URI, e.g. {@code "file:///tmp/polaris"} or {@code + * "s3://bucket/polaris"}. If the URI does not have a scheme, it will be assumed to be a local + * file URI. + */ + public static URI getTemporaryDirectory(Path defaultLocalDirectory) { + return getTemporaryDirectory(System::getenv, defaultLocalDirectory); + } + + @VisibleForTesting + static URI getTemporaryDirectory(Function getenv, Path defaultLocalDirectory) { + String envVar = getenv.apply(INTEGRATION_TEST_TEMP_DIR_ENV_VAR); + envVar = envVar != null ? envVar : defaultLocalDirectory.toString(); + envVar = envVar.startsWith("/") ? "file://" + envVar : envVar; + return URI.create(envVar + "/").normalize(); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 7a2961e402..48dbce059e 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -29,13 +29,12 @@ import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response.Status; import java.io.IOException; -import java.nio.file.Files; +import java.net.URI; import java.nio.file.Path; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionData; @@ -75,6 +74,7 @@ import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; import org.apache.polaris.service.it.env.RestApi; @@ -87,6 +87,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -109,7 +110,6 @@ public class PolarisApplicationIntegrationTest { public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; - private static Path testDir; private static String realm; private static RestApi managementApi; @@ -118,25 +118,22 @@ public class PolarisApplicationIntegrationTest { private static ClientCredentials clientCredentials; private static ClientPrincipal admin; private static String authToken; + private static URI baseLocation; private String principalRoleName; private String internalCatalogName; @BeforeAll - public static void setup(PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials) - throws IOException { + public static void setup( + PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials, @TempDir Path tempDir) { endpoints = apiEndpoints; client = polarisClient(endpoints); realm = endpoints.realmId(); admin = adminCredentials; clientCredentials = adminCredentials.credentials(); authToken = client.obtainToken(clientCredentials); - - testDir = Path.of("build/test_data/iceberg/" + realm); - FileUtils.deleteQuietly(testDir.toFile()); - Files.createDirectories(testDir); - managementApi = client.managementApi(clientCredentials); + baseLocation = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve(realm + "/"); } @AfterAll @@ -443,9 +440,9 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException { Catalog.TypeEnum.EXTERNAL, principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of(baseLocation.toString())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + baseLocation.toString()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -453,9 +450,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException { sessionCatalog.createNamespace(sessionContext, ns); TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); String location = - "file://" - + testDir.toFile().getAbsolutePath() - + "/testIcebergRegisterTableInExternalCatalog"; + baseLocation.resolve("testIcebergRegisterTableInExternalCatalog").toString(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; TableMetadata tableMetadata = @@ -489,19 +484,16 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException { Catalog.TypeEnum.EXTERNAL, principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of(baseLocation.toString())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + baseLocation.toString()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); - String location = - "file://" - + testDir.toFile().getAbsolutePath() - + "/testIcebergUpdateTableInExternalCatalog"; + String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get()); @@ -541,20 +533,16 @@ public void testIcebergDropTableInExternalCatalog() throws IOException { Catalog.TypeEnum.EXTERNAL, principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) - .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) + .setAllowedLocations(List.of(baseLocation.toString())) .build(), - "file://" + testDir.toFile().getAbsolutePath()); + baseLocation.toString()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table"); - String location = - "file://" - + testDir.toFile().getAbsolutePath() - + "/" - + "testIcebergDropTableInExternalCatalog"; + String location = baseLocation.resolve("testIcebergDropTableInExternalCatalog").toString(); String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; TableMetadata tableMetadata = diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index bc774c1cd8..505465d120 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -29,6 +29,8 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.reflect.Method; +import java.net.URI; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,6 +81,7 @@ import org.apache.polaris.service.it.env.CatalogApi; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.ManagementApi; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; @@ -92,6 +95,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; /** * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog @@ -108,9 +112,9 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) .orElse("arn:aws:iam::123456789012:role/my-role"); - private static final String S3_BUCKET_BASE = - Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) - .orElse("file:///tmp/buckets/my-bucket"); + + private static URI s3BucketBase; + private static URI externalCatalogBase; protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; private static String principalRoleName; @@ -125,7 +129,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private String currentCatalogName; private final String catalogBaseLocation = - S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; + s3BucketBase + "/" + System.getenv("USER") + "/path/to/data"; private static final String[] DEFAULT_CATALOG_PROPERTIES = { "allow.unstructured.table.location", "true", @@ -148,7 +152,8 @@ String[] properties() default { } @BeforeAll - static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + static void setup( + PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) { adminCredentials = credentials; endpoints = apiEndpoints; client = polarisClient(endpoints); @@ -157,6 +162,9 @@ static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credential principalRoleName = client.newEntityName("rest-admin"); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); catalogApi = client.catalogApi(principalCredentials); + URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir); + s3BucketBase = testRootUri.resolve("my-bucket"); + externalCatalogBase = testRootUri.resolve("external-catalog"); } @AfterAll @@ -192,7 +200,7 @@ public void before(TestInfo testInfo) { for (int i = 0; i < properties.length; i += 2) { catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); } - if (!S3_BUCKET_BASE.startsWith("file:/")) { + if (!s3BucketBase.getScheme().equals("file")) { catalogPropsBuilder.addProperty( CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); } @@ -202,7 +210,7 @@ public void before(TestInfo testInfo) { .setName(currentCatalogName) .setProperties(catalogPropsBuilder.build()) .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:/") + s3BucketBase.getScheme().equals("file") ? new FileStorageConfigInfo( StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) : awsConfigModel) @@ -541,12 +549,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + externalCatalogBase + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -576,12 +584,12 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + externalCatalogBase + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { @@ -610,12 +618,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled TableMetadata.newTableMetadata( new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))), PartitionSpec.unpartitioned(), - "file:///tmp/ns1/my_table", + externalCatalogBase + "/ns1/my_table", Map.of()); try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) { resolvingFileIO.initialize(Map.of()); resolvingFileIO.setConf(new Configuration()); - String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json"; + String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json"; TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation)); restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation); try { diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java index e7fa2dcece..3625c69e87 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java @@ -27,6 +27,7 @@ import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; +import java.net.URI; import java.nio.file.Path; import java.time.Instant; import java.util.List; @@ -41,6 +42,7 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.service.it.env.CatalogApi; import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IntegrationTestsHelper; import org.apache.polaris.service.it.env.ManagementApi; import org.apache.polaris.service.it.env.PolarisApiEndpoints; import org.apache.polaris.service.it.env.PolarisClient; @@ -82,7 +84,7 @@ public class PolarisSparkIntegrationTest { private String catalogName; private String externalCatalogName; - @TempDir public Path warehouseDir; + private URI warehouseDir; @BeforeAll public static void setup() throws IOException { @@ -95,13 +97,16 @@ public static void cleanup() { } @BeforeEach - public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + public void before( + PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) { endpoints = apiEndpoints; client = polarisClient(endpoints); sparkToken = client.obtainToken(credentials); managementApi = client.managementApi(credentials); catalogApi = client.catalogApi(credentials); + warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse"); + catalogName = client.newEntityName("spark_catalog"); externalCatalogName = client.newEntityName("spark_ext_catalog"); diff --git a/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java b/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java new file mode 100644 index 0000000000..ad65f7598d --- /dev/null +++ b/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class IntegrationTestsHelperTest { + + @ParameterizedTest + @MethodSource + void getTemporaryDirectory(String envVar, Path local, URI expected) { + URI actual = IntegrationTestsHelper.getTemporaryDirectory(s -> envVar, local); + assertThat(actual).isEqualTo(expected); + } + + static Stream getTemporaryDirectory() { + return Stream.of( + Arguments.of(null, Path.of("/tmp/polaris"), URI.create("file:///tmp/polaris/")), + Arguments.of(null, Path.of("/tmp/polaris/"), URI.create("file:///tmp/polaris/")), + Arguments.of( + "file:///tmp/polaris/from-env", + Path.of("/tmp/polaris/default"), + URI.create("file:///tmp/polaris/from-env/")), + Arguments.of( + "file:///tmp/polaris/from-env/", + Path.of("/tmp/polaris/default"), + URI.create("file:///tmp/polaris/from-env/")), + Arguments.of( + "/tmp/polaris/from-env", + Path.of("/tmp/polaris/default"), + URI.create("file:///tmp/polaris/from-env/")), + Arguments.of( + "/tmp/polaris/from-env/", + Path.of("/tmp/polaris/default"), + URI.create("file:///tmp/polaris/from-env/")), + Arguments.of( + "s3://bucket/polaris/from-env", + Path.of("/tmp/polaris/default"), + URI.create("s3://bucket/polaris/from-env/")), + Arguments.of( + "s3://bucket/polaris/from-env/", + Path.of("/tmp/polaris/default"), + URI.create("s3://bucket/polaris/from-env/"))); + } +}