diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts
index 6a2f587e50..47f6ab80f9 100644
--- a/integration-tests/build.gradle.kts
+++ b/integration-tests/build.gradle.kts
@@ -62,6 +62,7 @@ dependencies {
implementation(platform(libs.junit.bom))
implementation("org.junit.jupiter:junit-jupiter")
+ implementation("org.junit.jupiter:junit-jupiter-api")
compileOnly("org.junit.jupiter:junit-jupiter-engine")
implementation(libs.assertj.core)
implementation(libs.mockito.core)
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java
new file mode 100644
index 0000000000..7e2a8e41aa
--- /dev/null
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.env;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.function.Function;
+
+public final class IntegrationTestsHelper {
+
+ /**
+ * The environment variable that can be used to override the temporary directory used by the
+ * integration tests.
+ */
+ public static final String INTEGRATION_TEST_TEMP_DIR_ENV_VAR = "INTEGRATION_TEST_TEMP_DIR";
+
+ private IntegrationTestsHelper() {}
+
+ /**
+ * Get the temporary directory to use for integration tests.
+ *
+ *
If the environment variable {@link #INTEGRATION_TEST_TEMP_DIR_ENV_VAR} is set, it will be
+ * used as the temporary directory. Otherwise, the default local temporary directory will be used.
+ *
+ *
The environment variable should be a URI, e.g. {@code "file:///tmp/polaris"} or {@code
+ * "s3://bucket/polaris"}. If the URI does not have a scheme, it will be assumed to be a local
+ * file URI.
+ */
+ public static URI getTemporaryDirectory(Path defaultLocalDirectory) {
+ return getTemporaryDirectory(System::getenv, defaultLocalDirectory);
+ }
+
+ @VisibleForTesting
+ static URI getTemporaryDirectory(Function getenv, Path defaultLocalDirectory) {
+ String envVar = getenv.apply(INTEGRATION_TEST_TEMP_DIR_ENV_VAR);
+ envVar = envVar != null ? envVar : defaultLocalDirectory.toString();
+ envVar = envVar.startsWith("/") ? "file://" + envVar : envVar;
+ return URI.create(envVar + "/").normalize();
+ }
+}
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
index 7a2961e402..48dbce059e 100644
--- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
@@ -29,13 +29,12 @@
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.Response.Status;
import java.io.IOException;
-import java.nio.file.Files;
+import java.net.URI;
import java.nio.file.Path;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Map;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.BaseTable;
import org.apache.iceberg.PartitionData;
@@ -75,6 +74,7 @@
import org.apache.polaris.core.entity.PolarisEntityConstants;
import org.apache.polaris.service.it.env.ClientCredentials;
import org.apache.polaris.service.it.env.ClientPrincipal;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
import org.apache.polaris.service.it.env.PolarisApiEndpoints;
import org.apache.polaris.service.it.env.PolarisClient;
import org.apache.polaris.service.it.env.RestApi;
@@ -87,6 +87,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -109,7 +110,6 @@ public class PolarisApplicationIntegrationTest {
public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL";
- private static Path testDir;
private static String realm;
private static RestApi managementApi;
@@ -118,25 +118,22 @@ public class PolarisApplicationIntegrationTest {
private static ClientCredentials clientCredentials;
private static ClientPrincipal admin;
private static String authToken;
+ private static URI baseLocation;
private String principalRoleName;
private String internalCatalogName;
@BeforeAll
- public static void setup(PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials)
- throws IOException {
+ public static void setup(
+ PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials, @TempDir Path tempDir) {
endpoints = apiEndpoints;
client = polarisClient(endpoints);
realm = endpoints.realmId();
admin = adminCredentials;
clientCredentials = adminCredentials.credentials();
authToken = client.obtainToken(clientCredentials);
-
- testDir = Path.of("build/test_data/iceberg/" + realm);
- FileUtils.deleteQuietly(testDir.toFile());
- Files.createDirectories(testDir);
-
managementApi = client.managementApi(clientCredentials);
+ baseLocation = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve(realm + "/");
}
@AfterAll
@@ -443,9 +440,9 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
Catalog.TypeEnum.EXTERNAL,
principalRoleName,
FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
- .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+ .setAllowedLocations(List.of(baseLocation.toString()))
.build(),
- "file://" + testDir.toFile().getAbsolutePath());
+ baseLocation.toString());
try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
@@ -453,9 +450,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
sessionCatalog.createNamespace(sessionContext, ns);
TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
String location =
- "file://"
- + testDir.toFile().getAbsolutePath()
- + "/testIcebergRegisterTableInExternalCatalog";
+ baseLocation.resolve("testIcebergRegisterTableInExternalCatalog").toString();
String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
TableMetadata tableMetadata =
@@ -489,19 +484,16 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
Catalog.TypeEnum.EXTERNAL,
principalRoleName,
FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
- .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+ .setAllowedLocations(List.of(baseLocation.toString()))
.build(),
- "file://" + testDir.toFile().getAbsolutePath());
+ baseLocation.toString());
try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
Namespace ns = Namespace.of("db1");
sessionCatalog.createNamespace(sessionContext, ns);
TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
- String location =
- "file://"
- + testDir.toFile().getAbsolutePath()
- + "/testIcebergUpdateTableInExternalCatalog";
+ String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString();
String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get());
@@ -541,20 +533,16 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
Catalog.TypeEnum.EXTERNAL,
principalRoleName,
FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
- .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+ .setAllowedLocations(List.of(baseLocation.toString()))
.build(),
- "file://" + testDir.toFile().getAbsolutePath());
+ baseLocation.toString());
try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
Namespace ns = Namespace.of("db1");
sessionCatalog.createNamespace(sessionContext, ns);
TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
- String location =
- "file://"
- + testDir.toFile().getAbsolutePath()
- + "/"
- + "testIcebergDropTableInExternalCatalog";
+ String location = baseLocation.resolve("testIcebergDropTableInExternalCatalog").toString();
String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
TableMetadata tableMetadata =
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
index bc774c1cd8..505465d120 100644
--- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
@@ -29,6 +29,8 @@
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.reflect.Method;
+import java.net.URI;
+import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -79,6 +81,7 @@
import org.apache.polaris.service.it.env.CatalogApi;
import org.apache.polaris.service.it.env.ClientCredentials;
import org.apache.polaris.service.it.env.IcebergHelper;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
import org.apache.polaris.service.it.env.ManagementApi;
import org.apache.polaris.service.it.env.PolarisApiEndpoints;
import org.apache.polaris.service.it.env.PolarisClient;
@@ -92,6 +95,7 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
/**
* Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog
@@ -108,9 +112,9 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests
private static final String TEST_ROLE_ARN =
Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN"))
.orElse("arn:aws:iam::123456789012:role/my-role");
- private static final String S3_BUCKET_BASE =
- Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH"))
- .orElse("file:///tmp/buckets/my-bucket");
+
+ private static URI s3BucketBase;
+ private static URI externalCatalogBase;
protected static final String VIEW_QUERY = "select * from ns1.layer1_table";
private static String principalRoleName;
@@ -125,7 +129,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests
private String currentCatalogName;
private final String catalogBaseLocation =
- S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data";
+ s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
private static final String[] DEFAULT_CATALOG_PROPERTIES = {
"allow.unstructured.table.location", "true",
@@ -148,7 +152,8 @@ String[] properties() default {
}
@BeforeAll
- static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
+ static void setup(
+ PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) {
adminCredentials = credentials;
endpoints = apiEndpoints;
client = polarisClient(endpoints);
@@ -157,6 +162,9 @@ static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credential
principalRoleName = client.newEntityName("rest-admin");
principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
catalogApi = client.catalogApi(principalCredentials);
+ URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir);
+ s3BucketBase = testRootUri.resolve("my-bucket");
+ externalCatalogBase = testRootUri.resolve("external-catalog");
}
@AfterAll
@@ -192,7 +200,7 @@ public void before(TestInfo testInfo) {
for (int i = 0; i < properties.length; i += 2) {
catalogPropsBuilder.addProperty(properties[i], properties[i + 1]);
}
- if (!S3_BUCKET_BASE.startsWith("file:/")) {
+ if (!s3BucketBase.getScheme().equals("file")) {
catalogPropsBuilder.addProperty(
CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:");
}
@@ -202,7 +210,7 @@ public void before(TestInfo testInfo) {
.setName(currentCatalogName)
.setProperties(catalogPropsBuilder.build())
.setStorageConfigInfo(
- S3_BUCKET_BASE.startsWith("file:/")
+ s3BucketBase.getScheme().equals("file")
? new FileStorageConfigInfo(
StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://"))
: awsConfigModel)
@@ -541,12 +549,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable
TableMetadata.newTableMetadata(
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
PartitionSpec.unpartitioned(),
- "file:///tmp/ns1/my_table",
+ externalCatalogBase + "/ns1/my_table",
Map.of());
try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
resolvingFileIO.initialize(Map.of());
resolvingFileIO.setConf(new Configuration());
- String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
try {
@@ -576,12 +584,12 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa
TableMetadata.newTableMetadata(
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
PartitionSpec.unpartitioned(),
- "file:///tmp/ns1/my_table",
+ externalCatalogBase + "/ns1/my_table",
Map.of());
try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
resolvingFileIO.initialize(Map.of());
resolvingFileIO.setConf(new Configuration());
- String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
try {
@@ -610,12 +618,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled
TableMetadata.newTableMetadata(
new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
PartitionSpec.unpartitioned(),
- "file:///tmp/ns1/my_table",
+ externalCatalogBase + "/ns1/my_table",
Map.of());
try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
resolvingFileIO.initialize(Map.of());
resolvingFileIO.setConf(new Configuration());
- String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+ String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
try {
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
index e7fa2dcece..3625c69e87 100644
--- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
+++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
@@ -27,6 +27,7 @@
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.core.Response;
import java.io.IOException;
+import java.net.URI;
import java.nio.file.Path;
import java.time.Instant;
import java.util.List;
@@ -41,6 +42,7 @@
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.service.it.env.CatalogApi;
import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
import org.apache.polaris.service.it.env.ManagementApi;
import org.apache.polaris.service.it.env.PolarisApiEndpoints;
import org.apache.polaris.service.it.env.PolarisClient;
@@ -82,7 +84,7 @@ public class PolarisSparkIntegrationTest {
private String catalogName;
private String externalCatalogName;
- @TempDir public Path warehouseDir;
+ private URI warehouseDir;
@BeforeAll
public static void setup() throws IOException {
@@ -95,13 +97,16 @@ public static void cleanup() {
}
@BeforeEach
- public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
+ public void before(
+ PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) {
endpoints = apiEndpoints;
client = polarisClient(endpoints);
sparkToken = client.obtainToken(credentials);
managementApi = client.managementApi(credentials);
catalogApi = client.catalogApi(credentials);
+ warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse");
+
catalogName = client.newEntityName("spark_catalog");
externalCatalogName = client.newEntityName("spark_ext_catalog");
diff --git a/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java b/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java
new file mode 100644
index 0000000000..ad65f7598d
--- /dev/null
+++ b/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.env;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.stream.Stream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class IntegrationTestsHelperTest {
+
+ @ParameterizedTest
+ @MethodSource
+ void getTemporaryDirectory(String envVar, Path local, URI expected) {
+ URI actual = IntegrationTestsHelper.getTemporaryDirectory(s -> envVar, local);
+ assertThat(actual).isEqualTo(expected);
+ }
+
+ static Stream getTemporaryDirectory() {
+ return Stream.of(
+ Arguments.of(null, Path.of("/tmp/polaris"), URI.create("file:///tmp/polaris/")),
+ Arguments.of(null, Path.of("/tmp/polaris/"), URI.create("file:///tmp/polaris/")),
+ Arguments.of(
+ "file:///tmp/polaris/from-env",
+ Path.of("/tmp/polaris/default"),
+ URI.create("file:///tmp/polaris/from-env/")),
+ Arguments.of(
+ "file:///tmp/polaris/from-env/",
+ Path.of("/tmp/polaris/default"),
+ URI.create("file:///tmp/polaris/from-env/")),
+ Arguments.of(
+ "/tmp/polaris/from-env",
+ Path.of("/tmp/polaris/default"),
+ URI.create("file:///tmp/polaris/from-env/")),
+ Arguments.of(
+ "/tmp/polaris/from-env/",
+ Path.of("/tmp/polaris/default"),
+ URI.create("file:///tmp/polaris/from-env/")),
+ Arguments.of(
+ "s3://bucket/polaris/from-env",
+ Path.of("/tmp/polaris/default"),
+ URI.create("s3://bucket/polaris/from-env/")),
+ Arguments.of(
+ "s3://bucket/polaris/from-env/",
+ Path.of("/tmp/polaris/default"),
+ URI.create("s3://bucket/polaris/from-env/")));
+ }
+}