Integration tests: improve support for temporary folders (#987)

adutra · web-flow · commit 839bc2573fb8 · 2025-02-25T12:01:13.000+01:00
3 integration tests currently require a temporary folder for various purposes. The folders are generally hard-coded to some local filesystem location, which makes those tests impossible to execute against a remote Polaris instance.

This PR aims at fixing this situation by leveraging a new optional environment variable that points to a location that is accessible by both the server and the client running the tests. This would typically be an S3 bucket, for example, but could also be a shared volume mounted on both machines at the same mount point. If this environment variable is not present, then local execution is assumed and a temporary directory managed by JUnit is used instead.
diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts
@@ -62,6 +62,7 @@ dependencies {
 
   implementation(platform(libs.junit.bom))
   implementation("org.junit.jupiter:junit-jupiter")
+  implementation("org.junit.jupiter:junit-jupiter-api")
   compileOnly("org.junit.jupiter:junit-jupiter-engine")
   implementation(libs.assertj.core)
   implementation(libs.mockito.core)
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IntegrationTestsHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.it.env;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.function.Function;
+
+public final class IntegrationTestsHelper {
+
+  /**
+   * The environment variable that can be used to override the temporary directory used by the
+   * integration tests.
+   */
+  public static final String INTEGRATION_TEST_TEMP_DIR_ENV_VAR = "INTEGRATION_TEST_TEMP_DIR";
+
+  private IntegrationTestsHelper() {}
+
+  /**
+   * Get the temporary directory to use for integration tests.
+   *
+   * <p>If the environment variable {@link #INTEGRATION_TEST_TEMP_DIR_ENV_VAR} is set, it will be
+   * used as the temporary directory. Otherwise, the default local temporary directory will be used.
+   *
+   * <p>The environment variable should be a URI, e.g. {@code "file:///tmp/polaris"} or {@code
+   * "s3://bucket/polaris"}. If the URI does not have a scheme, it will be assumed to be a local
+   * file URI.
+   */
+  public static URI getTemporaryDirectory(Path defaultLocalDirectory) {
+    return getTemporaryDirectory(System::getenv, defaultLocalDirectory);
+  }
+
+  @VisibleForTesting
+  static URI getTemporaryDirectory(Function<String, String> getenv, Path defaultLocalDirectory) {
+    String envVar = getenv.apply(INTEGRATION_TEST_TEMP_DIR_ENV_VAR);
+    envVar = envVar != null ? envVar : defaultLocalDirectory.toString();
+    envVar = envVar.startsWith("/") ? "file://" + envVar : envVar;
+    return URI.create(envVar + "/").normalize();
+  }
+}
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java
@@ -29,13 +29,12 @@
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.core.Response.Status;
 import java.io.IOException;
-import java.nio.file.Files;
+import java.net.URI;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.iceberg.BaseTable;
 import org.apache.iceberg.PartitionData;
@@ -75,6 +74,7 @@
 import org.apache.polaris.core.entity.PolarisEntityConstants;
 import org.apache.polaris.service.it.env.ClientCredentials;
 import org.apache.polaris.service.it.env.ClientPrincipal;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
 import org.apache.polaris.service.it.env.PolarisApiEndpoints;
 import org.apache.polaris.service.it.env.PolarisClient;
 import org.apache.polaris.service.it.env.RestApi;
@@ -87,6 +87,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
@@ -109,7 +110,6 @@ public class PolarisApplicationIntegrationTest {
 
   public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL";
 
-  private static Path testDir;
   private static String realm;
 
   private static RestApi managementApi;
@@ -118,25 +118,22 @@ public class PolarisApplicationIntegrationTest {
   private static ClientCredentials clientCredentials;
   private static ClientPrincipal admin;
   private static String authToken;
+  private static URI baseLocation;
 
   private String principalRoleName;
   private String internalCatalogName;
 
   @BeforeAll
-  public static void setup(PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials)
-      throws IOException {
+  public static void setup(
+      PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials, @TempDir Path tempDir) {
     endpoints = apiEndpoints;
     client = polarisClient(endpoints);
     realm = endpoints.realmId();
     admin = adminCredentials;
     clientCredentials = adminCredentials.credentials();
     authToken = client.obtainToken(clientCredentials);
-
-    testDir = Path.of("build/test_data/iceberg/" + realm);
-    FileUtils.deleteQuietly(testDir.toFile());
-    Files.createDirectories(testDir);
-
     managementApi = client.managementApi(clientCredentials);
+    baseLocation = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve(realm + "/");
   }
 
   @AfterAll
@@ -443,19 +440,17 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
         Catalog.TypeEnum.EXTERNAL,
         principalRoleName,
         FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
-            .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+            .setAllowedLocations(List.of(baseLocation.toString()))
             .build(),
-        "file://" + testDir.toFile().getAbsolutePath());
+        baseLocation.toString());
     try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
         HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
       SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
       Namespace ns = Namespace.of("db1");
       sessionCatalog.createNamespace(sessionContext, ns);
       TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
       String location =
-          "file://"
-              + testDir.toFile().getAbsolutePath()
-              + "/testIcebergRegisterTableInExternalCatalog";
+          baseLocation.resolve("testIcebergRegisterTableInExternalCatalog").toString();
       String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
 
       TableMetadata tableMetadata =
@@ -489,19 +484,16 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
         Catalog.TypeEnum.EXTERNAL,
         principalRoleName,
         FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
-            .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+            .setAllowedLocations(List.of(baseLocation.toString()))
             .build(),
-        "file://" + testDir.toFile().getAbsolutePath());
+        baseLocation.toString());
     try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
         HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
       SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
       Namespace ns = Namespace.of("db1");
       sessionCatalog.createNamespace(sessionContext, ns);
       TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
-      String location =
-          "file://"
-              + testDir.toFile().getAbsolutePath()
-              + "/testIcebergUpdateTableInExternalCatalog";
+      String location = baseLocation.resolve("testIcebergUpdateTableInExternalCatalog").toString();
       String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
 
       Types.NestedField col1 = Types.NestedField.of(1, false, "col1", Types.StringType.get());
@@ -541,20 +533,16 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
         Catalog.TypeEnum.EXTERNAL,
         principalRoleName,
         FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
-            .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath()))
+            .setAllowedLocations(List.of(baseLocation.toString()))
             .build(),
-        "file://" + testDir.toFile().getAbsolutePath());
+        baseLocation.toString());
     try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName);
         HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) {
       SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
       Namespace ns = Namespace.of("db1");
       sessionCatalog.createNamespace(sessionContext, ns);
       TableIdentifier tableIdentifier = TableIdentifier.of(ns, "the_table");
-      String location =
-          "file://"
-              + testDir.toFile().getAbsolutePath()
-              + "/"
-              + "testIcebergDropTableInExternalCatalog";
+      String location = baseLocation.resolve("testIcebergDropTableInExternalCatalog").toString();
       String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json";
 
       TableMetadata tableMetadata =
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java
@@ -29,6 +29,8 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.reflect.Method;
+import java.net.URI;
+import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -79,6 +81,7 @@
 import org.apache.polaris.service.it.env.CatalogApi;
 import org.apache.polaris.service.it.env.ClientCredentials;
 import org.apache.polaris.service.it.env.IcebergHelper;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
 import org.apache.polaris.service.it.env.ManagementApi;
 import org.apache.polaris.service.it.env.PolarisApiEndpoints;
 import org.apache.polaris.service.it.env.PolarisClient;
@@ -92,6 +95,7 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.io.TempDir;
 
 /**
  * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog
@@ -108,9 +112,9 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
   private static final String TEST_ROLE_ARN =
       Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN"))
           .orElse("arn:aws:iam::123456789012:role/my-role");
-  private static final String S3_BUCKET_BASE =
-      Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH"))
-          .orElse("file:///tmp/buckets/my-bucket");
+
+  private static URI s3BucketBase;
+  private static URI externalCatalogBase;
 
   protected static final String VIEW_QUERY = "select * from ns1.layer1_table";
   private static String principalRoleName;
@@ -125,7 +129,7 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
   private String currentCatalogName;
 
   private final String catalogBaseLocation =
-      S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data";
+      s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
 
   private static final String[] DEFAULT_CATALOG_PROPERTIES = {
     "allow.unstructured.table.location", "true",
@@ -148,7 +152,8 @@ String[] properties() default {
   }
 
   @BeforeAll
-  static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
+  static void setup(
+      PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) {
     adminCredentials = credentials;
     endpoints = apiEndpoints;
     client = polarisClient(endpoints);
@@ -157,6 +162,9 @@ static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credential
     principalRoleName = client.newEntityName("rest-admin");
     principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
     catalogApi = client.catalogApi(principalCredentials);
+    URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir);
+    s3BucketBase = testRootUri.resolve("my-bucket");
+    externalCatalogBase = testRootUri.resolve("external-catalog");
   }
 
   @AfterAll
@@ -192,7 +200,7 @@ public void before(TestInfo testInfo) {
     for (int i = 0; i < properties.length; i += 2) {
       catalogPropsBuilder.addProperty(properties[i], properties[i + 1]);
     }
-    if (!S3_BUCKET_BASE.startsWith("file:/")) {
+    if (!s3BucketBase.getScheme().equals("file")) {
       catalogPropsBuilder.addProperty(
           CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:");
     }
@@ -202,7 +210,7 @@ public void before(TestInfo testInfo) {
             .setName(currentCatalogName)
             .setProperties(catalogPropsBuilder.build())
             .setStorageConfigInfo(
-                S3_BUCKET_BASE.startsWith("file:/")
+                s3BucketBase.getScheme().equals("file")
                     ? new FileStorageConfigInfo(
                         StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://"))
                     : awsConfigModel)
@@ -541,12 +549,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigDisable
         TableMetadata.newTableMetadata(
             new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
             PartitionSpec.unpartitioned(),
-            "file:///tmp/ns1/my_table",
+            externalCatalogBase + "/ns1/my_table",
             Map.of());
     try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
       resolvingFileIO.initialize(Map.of());
       resolvingFileIO.setConf(new Configuration());
-      String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+      String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
       TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
       restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
       try {
@@ -576,12 +584,12 @@ public void testLoadTableWithoutAccessDelegationForExternalCatalogWithConfigDisa
         TableMetadata.newTableMetadata(
             new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
             PartitionSpec.unpartitioned(),
-            "file:///tmp/ns1/my_table",
+            externalCatalogBase + "/ns1/my_table",
             Map.of());
     try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
       resolvingFileIO.initialize(Map.of());
       resolvingFileIO.setConf(new Configuration());
-      String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+      String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
       TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
       restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
       try {
@@ -610,12 +618,12 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled
         TableMetadata.newTableMetadata(
             new Schema(List.of(Types.NestedField.of(1, false, "col1", new Types.StringType()))),
             PartitionSpec.unpartitioned(),
-            "file:///tmp/ns1/my_table",
+            externalCatalogBase + "/ns1/my_table",
             Map.of());
     try (ResolvingFileIO resolvingFileIO = new ResolvingFileIO()) {
       resolvingFileIO.initialize(Map.of());
       resolvingFileIO.setConf(new Configuration());
-      String fileLocation = "file:///tmp/ns1/my_table/metadata/v1.metadata.json";
+      String fileLocation = externalCatalogBase + "/ns1/my_table/metadata/v1.metadata.json";
       TableMetadataParser.write(tableMetadata, resolvingFileIO.newOutputFile(fileLocation));
       restCatalog.registerTable(TableIdentifier.of(ns1, "my_table"), fileLocation);
       try {
diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java
@@ -27,6 +27,7 @@
 import jakarta.ws.rs.client.Entity;
 import jakarta.ws.rs.core.Response;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.List;
@@ -41,6 +42,7 @@
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
 import org.apache.polaris.service.it.env.CatalogApi;
 import org.apache.polaris.service.it.env.ClientCredentials;
+import org.apache.polaris.service.it.env.IntegrationTestsHelper;
 import org.apache.polaris.service.it.env.ManagementApi;
 import org.apache.polaris.service.it.env.PolarisApiEndpoints;
 import org.apache.polaris.service.it.env.PolarisClient;
@@ -82,7 +84,7 @@ public class PolarisSparkIntegrationTest {
   private String catalogName;
   private String externalCatalogName;
 
-  @TempDir public Path warehouseDir;
+  private URI warehouseDir;
 
   @BeforeAll
   public static void setup() throws IOException {
@@ -95,13 +97,16 @@ public static void cleanup() {
   }
 
   @BeforeEach
-  public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) {
+  public void before(
+      PolarisApiEndpoints apiEndpoints, ClientCredentials credentials, @TempDir Path tempDir) {
     endpoints = apiEndpoints;
     client = polarisClient(endpoints);
     sparkToken = client.obtainToken(credentials);
     managementApi = client.managementApi(credentials);
     catalogApi = client.catalogApi(credentials);
 
+    warehouseDir = IntegrationTestsHelper.getTemporaryDirectory(tempDir).resolve("spark-warehouse");
+
     catalogName = client.newEntityName("spark_catalog");
     externalCatalogName = client.newEntityName("spark_ext_catalog");
 
diff --git a/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java b/integration-tests/src/test/java/org/apache/polaris/service/it/env/IntegrationTestsHelperTest.java
