Upgrade Iceberg to 1.8

getting-started/spark/notebooks/SparkPolaris.ipynb

@@ -256,7 +256,7 @@
     "\n",
     "spark = (SparkSession.builder\n",
     "  .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.iceberg.spark.SparkSessionCatalog\")\n",
-    "  .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
+    "  .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n",
     "  .config('spark.sql.iceberg.vectorization.enabled', 'false')\n",
     "         \n",
     "  # Configure the 'polaris' catalog as an Iceberg rest catalog\n",

gradle/libs.versions.toml

@@ -19,7 +19,7 @@
 
 [versions]
 hadoop = "3.4.1"
-iceberg = "1.7.1"
+iceberg = "1.8.1" # Ensure to update the iceberg version in regtests to keep regtests up-to-date
 quarkus = "3.19.4"
 immutables = "2.10.1"
 picocli = "4.7.6"

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java

@@ -454,8 +454,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(
-                  new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1)
+              .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 
@@ -498,7 +497,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(new Schema(col1), 1)
+              .addSchema(new Schema(col1))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 
@@ -546,8 +545,7 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(
-                  new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1)
+              .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java

@@ -76,7 +76,6 @@
 import org.apache.polaris.core.admin.model.ViewGrant;
 import org.apache.polaris.core.admin.model.ViewPrivilege;
 import org.apache.polaris.core.config.FeatureConfiguration;
-import org.apache.polaris.core.config.PolarisConfiguration;
 import org.apache.polaris.core.entity.CatalogEntity;
 import org.apache.polaris.core.entity.PolarisEntityConstants;
 import org.apache.polaris.service.it.env.CatalogApi;
@@ -105,7 +104,7 @@
  * @implSpec @implSpec This test expects the server to be configured with the following features
  *     configured:
  *     <ul>
- *       <li>{@link PolarisConfiguration#ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING}: {@code false}
+ *       <li>{@link FeatureConfiguration#ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING}: {@code false}
  *     </ul>
  */
 @ExtendWith(PolarisIntegrationTestExtension.class)
@@ -118,16 +117,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
   private static URI externalCatalogBase;
 
   protected static final String VIEW_QUERY = "select * from ns1.layer1_table";
-  private static String principalRoleName;
   private static ClientCredentials adminCredentials;
-  private static PrincipalWithCredentials principalCredentials;
   private static PolarisApiEndpoints endpoints;
   private static PolarisClient client;
   private static ManagementApi managementApi;
-  private static CatalogApi catalogApi;
 
+  private PrincipalWithCredentials principalCredentials;
+  private CatalogApi catalogApi;
   private RESTCatalog restCatalog;
   private String currentCatalogName;
+  private Map<String, String> restCatalogConfig;
 
   private final String catalogBaseLocation =
       s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
@@ -159,10 +158,6 @@ static void setup(
     endpoints = apiEndpoints;
     client = polarisClient(endpoints);
     managementApi = client.managementApi(credentials);
-    String principalName = client.newEntityName("snowman-rest");
-    principalRoleName = client.newEntityName("rest-admin");
-    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
-    catalogApi = client.catalogApi(principalCredentials);
     URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir);
     s3BucketBase = testRootUri.resolve("my-bucket");
     externalCatalogBase = testRootUri.resolve("external-catalog");
@@ -175,10 +170,9 @@ static void close() throws Exception {
 
   @BeforeEach
   public void before(TestInfo testInfo) {
-    String principalName = "snowman-rest-" + UUID.randomUUID();
-    principalRoleName = "rest-admin-" + UUID.randomUUID();
-    PrincipalWithCredentials principalCredentials =
-        managementApi.createPrincipalWithRole(principalName, principalRoleName);
+    String principalName = client.newEntityName("snowman-rest");
+    String principalRoleName = client.newEntityName("rest-admin");
+    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
 
     catalogApi = client.catalogApi(principalCredentials);
 
@@ -219,29 +213,26 @@ public void before(TestInfo testInfo) {
 
     managementApi.createCatalog(principalRoleName, catalog);
 
-    Optional<PolarisRestCatalogIntegrationTest.RestCatalogConfig> restCatalogConfig =
+    restCatalogConfig =
         testInfo
             .getTestMethod()
-            .flatMap(
-                m ->
-                    Optional.ofNullable(
-                        m.getAnnotation(
-                            PolarisRestCatalogIntegrationTest.RestCatalogConfig.class)));
-    ImmutableMap.Builder<String, String> extraPropertiesBuilder = ImmutableMap.builder();
-    restCatalogConfig.ifPresent(
-        config -> {
-          for (int i = 0; i < config.value().length; i += 2) {
-            extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]);
-          }
-        });
-
-    restCatalog =
-        IcebergHelper.restCatalog(
-            client,
-            endpoints,
-            principalCredentials,
-            currentCatalogName,
-            extraPropertiesBuilder.build());
+            .map(m -> m.getAnnotation(RestCatalogConfig.class))
+            .map(RestCatalogConfig::value)
+            .map(
+                values -> {
+                  if (values.length % 2 != 0) {
+                    throw new IllegalArgumentException(
+                        String.format("Missing value for config '%s'", values[values.length - 1]));
+                  }
+                  Map<String, String> config = new HashMap<>();
+                  for (int i = 0; i < values.length; i += 2) {
+                    config.put(values[i], values[i + 1]);
+                  }
+                  return config;
+                })
+            .orElse(ImmutableMap.of());
+
+    restCatalog = initCatalog(currentCatalogName, ImmutableMap.of());
   }
 
   @AfterEach
@@ -254,6 +245,26 @@ protected RESTCatalog catalog() {
     return restCatalog;
   }
 
+  /**
+   * Initialize a RESTCatalog for testing.
+   *
+   * @param catalogName this parameter is currently unused.
+   * @param additionalProperties additional properties to apply on top of the default test settings
+   * @return a configured instance of RESTCatalog
+   */
+  @Override
+  protected RESTCatalog initCatalog(String catalogName, Map<String, String> additionalProperties) {
+    ImmutableMap.Builder<String, String> extraPropertiesBuilder = ImmutableMap.builder();
+    extraPropertiesBuilder.putAll(restCatalogConfig);
+    extraPropertiesBuilder.putAll(additionalProperties);
+    return IcebergHelper.restCatalog(
+        client,
+        endpoints,
+        principalCredentials,
+        currentCatalogName,
+        extraPropertiesBuilder.buildKeepingLast());
+  }
+
   @Override
   protected boolean requiresNamespaceCreate() {
     return true;

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java

@@ -42,6 +42,8 @@
 import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 
@@ -113,7 +115,16 @@ public void before(TestInfo testInfo) {
     managementApi.createCatalog(principalRoleName, catalog);
 
     restCatalog =
-        IcebergHelper.restCatalog(client, endpoints, principalCredentials, catalogName, Map.of());
+        IcebergHelper.restCatalog(
+            client,
+            endpoints,
+            principalCredentials,
+            catalogName,
+            Map.of(
+                org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
+                "catalog-default-key1",
+                org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
+                "catalog-default-key2"));
   }
 
   @AfterEach
@@ -156,4 +167,16 @@ protected boolean supportsServerSideRetry() {
   protected boolean overridesRequestedLocation() {
     return true;
   }
+
+  /** TODO: Unblock this test, see: https://github.com/apache/polaris/issues/1273 */
+  @Override
+  @Test
+  @Disabled(
+      """
+      Disabled because the behavior is not applicable to Polaris.
+      To unblock, update this to expect an exception and add a Polaris-specific test.
+      """)
+  public void createViewWithCustomMetadataLocation() {
+    super.createViewWithCustomMetadataLocation();
+  }
 }

quarkus/service/build.gradle.kts

@@ -23,6 +23,11 @@ plugins {
   id("polaris-quarkus")
 }
 
+configurations.all {
+  // exclude junit4 dependency for this module
+  exclude(group = "junit", module = "junit")
+}
+
 dependencies {
   implementation(project(":polaris-core"))
   implementation(project(":polaris-api-management-service"))

quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/IcebergCatalogHandlerWrapperAuthzTest.java

@@ -1721,7 +1721,7 @@ public Catalog createCallContextCatalog(
             FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new Configuration());
             TableMetadata tableMetadata =
                 TableMetadata.buildFromEmpty()
-                    .addSchema(SCHEMA, SCHEMA.highestFieldId())
+                    .addSchema(SCHEMA)
                     .setLocation(
                         String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation))
                     .addPartitionSpec(PartitionSpec.unpartitioned())