merge from main

Author: adnanhemani

.github/workflows/python-client.yml

@@ -51,7 +51,7 @@ jobs:
           java-version: '21'
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6
         with:
           python-version: ${{ matrix.python-version }}
 

Makefile

@@ -140,7 +140,7 @@ client-regenerate: client-setup-env ## Regenerate the client code
 .PHONY: client-unit-test
 client-unit-test: client-setup-env ## Run client unit tests
 	@echo "--- Running client unit tests ---"
-	@$(ACTIVATE_AND_CD) && SCRIPT_DIR="non-existing-mock-directory" poetry run pytest test/
+	@$(ACTIVATE_AND_CD) && poetry run pytest test/
 	@echo "--- Client unit tests complete ---"
 
 .PHONY: client-integration-test

client/python/cli/command/profiles.py

@@ -56,6 +56,7 @@ def _load_profiles(self) -> Dict[str, Dict[str, str]]:
         if not os.path.exists(CONFIG_FILE):
             return {}
         with open(CONFIG_FILE, "r") as f:
+            print(f"Loading profiles from {CONFIG_FILE}")
             return json.load(f)
 
     def _save_profiles(self, profiles: Dict[str, Dict[str, str]]) -> None:

client/python/cli/constants.py

@@ -371,9 +371,5 @@ class Namespaces:
 CLIENT_PROFILE_ENV = "CLIENT_PROFILE"
 DEFAULT_HOSTNAME = "localhost"
 DEFAULT_PORT = 8181
-CONFIG_DIR = os.environ.get("SCRIPT_DIR")
-if CONFIG_DIR is None:
-    raise Exception(
-        "The SCRIPT_DIR environment variable is not set. Please set it to the Polaris's script directory."
-    )
+CONFIG_DIR = (os.environ.get("POLARIS_HOME") or "").strip() or os.path.expanduser("~/.polaris")
 CONFIG_FILE = os.path.join(CONFIG_DIR, ".polaris.json")

getting-started/telemetry/docker-compose.yml

@@ -85,7 +85,7 @@ services:
 
   # Jaeger (OpenTelemetry traces collector)
   jaeger:
-    image: docker.io/jaegertracing/all-in-one:1.72.0
+    image: docker.io/jaegertracing/all-in-one:1.73.0
     ports:
       # Jaeger gRPC collector, used by Polaris
       - "4317:4317"

gradle/libs.versions.toml

@@ -44,9 +44,8 @@ assertj-core = { module = "org.assertj:assertj-core", version = "3.27.4" }
 auth0-jwt = { module = "com.auth0:java-jwt", version = "4.5.0" }
 awssdk-bom = { module = "software.amazon.awssdk:bom", version = "2.33.0" }
 awaitility = { module = "org.awaitility:awaitility", version = "4.3.0" }
-azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.37" }
+azuresdk-bom = { module = "com.azure:azure-sdk-bom", version = "1.2.38" }
 caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "3.2.2" }
-commons-codec = { module = "commons-codec:commons-codec", version = "1.19.0" }
 commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.18.0" }
 commons-text = { module = "org.apache.commons:commons-text", version = "1.14.0" }
 eclipselink = { module = "org.eclipse.persistence:eclipselink", version = "4.0.7" }

integration-tests/src/main/java/org/apache/polaris/service/it/ext/SparkSessionBuilder.java

@@ -164,13 +164,9 @@ private void applySingleCatalogConfig(CatalogConfig catalog) {
 
     // Add endpoint configuration
     Preconditions.checkNotNull(catalog.endpoints, "endpoints is required");
-    builder
-        .config(
-            String.format("spark.sql.catalog.%s.uri", catalog.catalogName),
-            catalog.endpoints.catalogApiEndpoint().toString())
-        .config(
-            String.format("spark.sql.catalog.%s.header.realm", catalog.catalogName),
-            catalog.endpoints.realmId());
+    builder.config(
+        String.format("spark.sql.catalog.%s.uri", catalog.catalogName),
+        catalog.endpoints.catalogApiEndpoint().toString());
 
     // Add token configuration
     if (catalog.token != null) {

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java

@@ -20,9 +20,11 @@
 
 import static org.apache.polaris.service.it.env.PolarisClient.polarisClient;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.awaitility.Awaitility.await;
 
+import com.google.common.collect.ImmutableMap;
 import jakarta.ws.rs.ProcessingException;
 import jakarta.ws.rs.client.Entity;
 import jakarta.ws.rs.client.Invocation;
@@ -67,6 +69,7 @@
 import org.apache.polaris.core.admin.model.PolarisCatalog;
 import org.apache.polaris.core.admin.model.PrincipalRole;
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
+import org.apache.polaris.core.config.BehaviorChangeConfiguration;
 import org.apache.polaris.core.entity.CatalogEntity;
 import org.apache.polaris.core.entity.PolarisEntityConstants;
 import org.apache.polaris.service.it.env.ClientPrincipal;
@@ -76,6 +79,7 @@
 import org.apache.polaris.service.it.env.RestApi;
 import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension;
 import org.assertj.core.api.InstanceOfAssertFactories;
+import org.assertj.core.api.ThrowableAssert;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
@@ -84,6 +88,8 @@
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 /**
  * @implSpec This test expects the server to be configured with the following features configured:
@@ -186,11 +192,30 @@ private static void createCatalog(
       String principalRoleName,
       StorageConfigInfo storageConfig,
       String defaultBaseLocation) {
-    CatalogProperties props =
+    createCatalog(
+        catalogName,
+        catalogType,
+        principalRoleName,
+        storageConfig,
+        defaultBaseLocation,
+        ImmutableMap.of());
+  }
+
+  private static void createCatalog(
+      String catalogName,
+      Catalog.TypeEnum catalogType,
+      String principalRoleName,
+      StorageConfigInfo storageConfig,
+      String defaultBaseLocation,
+      Map<String, String> additionalProperties) {
+    CatalogProperties.Builder propsBuilder =
         CatalogProperties.builder(defaultBaseLocation)
             .addProperty(
-                CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:/")
-            .build();
+                CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:/");
+    for (var entry : additionalProperties.entrySet()) {
+      propsBuilder.addProperty(entry.getKey(), entry.getValue());
+    }
+    CatalogProperties props = propsBuilder.build();
     Catalog catalog =
         catalogType.equals(Catalog.TypeEnum.INTERNAL)
             ? PolarisCatalog.builder()
@@ -641,4 +666,53 @@ public void testRequestBodyTooLarge() throws Exception {
               });
     }
   }
+
+  @ParameterizedTest
+  @ValueSource(booleans = {true, false})
+  public void testNamespaceOutsideCatalog(boolean allowNamespaceLocationEscape) throws IOException {
+    String catalogName = client.newEntityName("testNamespaceOutsideCatalog_specificLocation");
+    String catalogLocation = baseLocation.resolve(catalogName + "/catalog").toString();
+    String badLocation = baseLocation.resolve(catalogName + "/ns").toString();
+    createCatalog(
+        catalogName,
+        Catalog.TypeEnum.INTERNAL,
+        principalRoleName,
+        FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE)
+            .setAllowedLocations(List.of(catalogLocation))
+            .build(),
+        catalogLocation,
+        ImmutableMap.of(
+            BehaviorChangeConfiguration.ALLOW_NAMESPACE_CUSTOM_LOCATION.catalogConfig(),
+            String.valueOf(allowNamespaceLocationEscape)));
+    try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) {
+      SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+      sessionCatalog.createNamespace(sessionContext, Namespace.of("good_namespace"));
+      ThrowableAssert.ThrowingCallable createBadNamespace =
+          () ->
+              sessionCatalog.createNamespace(
+                  sessionContext,
+                  Namespace.of("bad_namespace"),
+                  ImmutableMap.of("location", badLocation));
+      if (!allowNamespaceLocationEscape) {
+        assertThatThrownBy(createBadNamespace)
+            .isInstanceOf(BadRequestException.class)
+            .hasMessageContaining("custom location");
+      } else {
+        assertThatCode(createBadNamespace).doesNotThrowAnyException();
+      }
+      ThrowableAssert.ThrowingCallable createBadChildGoodParent =
+          () ->
+              sessionCatalog.createNamespace(
+                  sessionContext,
+                  Namespace.of("good_namespace", "bad_child"),
+                  ImmutableMap.of("location", badLocation));
+      if (!allowNamespaceLocationEscape) {
+        assertThatThrownBy(createBadChildGoodParent)
+            .isInstanceOf(BadRequestException.class)
+            .hasMessageContaining("custom location");
+      } else {
+        assertThatCode(createBadChildGoodParent).doesNotThrowAnyException();
+      }
+    }
+  }
 }

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationBase.java

@@ -173,7 +173,8 @@ public abstract class PolarisRestCatalogIntegrationBase extends CatalogTests<RES
       Map.of(
           "polaris.config.allow.unstructured.table.location", "true",
           "polaris.config.allow.external.table.location", "true",
-          "polaris.config.list-pagination-enabled", "true");
+          "polaris.config.list-pagination-enabled", "true",
+          "polaris.config.namespace-custom-location.enabled", "true");
 
   /**
    * Get the storage configuration information for the catalog.

plugins/spark/v3.5/integration/src/intTest/java/org/apache/polaris/spark/quarkus/it/SparkIntegrationBase.java

@@ -100,6 +100,7 @@ public void before(
     CatalogProperties props = new CatalogProperties("s3://my-bucket/path/to/data");
     props.putAll(s3Container.getS3ConfigProperties());
     props.put("polaris.config.drop-with-purge.enabled", "true");
+    props.put("polaris.config.namespace-custom-location.enabled", "true");
     Catalog catalog =
         PolarisCatalog.builder()
             .setType(Catalog.TypeEnum.INTERNAL)