From 54562f2c389ce1c80eb3d1b01500f3ada52ee62f Mon Sep 17 00:00:00 2001 From: Yong Date: Mon, 28 Apr 2025 23:44:17 -0500 Subject: [PATCH 1/3] Additional refs update for iceberg 1.9.0 --- getting-started/spark/notebooks/SparkPolaris.ipynb | 2 +- plugins/pluginlibs.versions.toml | 2 +- plugins/spark/README.md | 14 +++++++------- quarkus/admin/distribution/LICENSE | 14 +++++++------- quarkus/server/distribution/LICENSE | 14 +++++++------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb b/getting-started/spark/notebooks/SparkPolaris.ipynb index 48fff32ace..42d56a87ea 100644 --- a/getting-started/spark/notebooks/SparkPolaris.ipynb +++ b/getting-started/spark/notebooks/SparkPolaris.ipynb @@ -256,7 +256,7 @@ "\n", "spark = (SparkSession.builder\n", " .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.iceberg.spark.SparkSessionCatalog\")\n", - " .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n", + " .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0,software.amazon.awssdk:bundle:2.23.19,software.amazon.awssdk:url-connection-client:2.23.19\")\n", " .config('spark.sql.iceberg.vectorization.enabled', 'false')\n", " \n", " # Configure the 'polaris' catalog as an Iceberg rest catalog\n", diff --git a/plugins/pluginlibs.versions.toml b/plugins/pluginlibs.versions.toml index 4f7288ff64..0ca3ba5d32 100644 --- a/plugins/pluginlibs.versions.toml +++ b/plugins/pluginlibs.versions.toml @@ -18,7 +18,7 @@ # [versions] -iceberg = "1.8.1" +iceberg = "1.9.0" spark35 = "3.5.5" scala212 = "2.12.19" scala213 = "2.13.15" diff --git a/plugins/spark/README.md b/plugins/spark/README.md index 6386a914c0..aa934cac6f 100644 --- a/plugins/spark/README.md +++ b/plugins/spark/README.md @@ -6,9 +6,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -20,19 +20,19 @@ # Polaris Spark Plugin The Polaris Spark plugin provides a SparkCatalog class, which communicates with the Polaris -REST endpoints, and provides implementations for Apache Spark's -[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java), -[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java), +REST endpoints, and provides implementations for Apache Spark's +[TableCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/TableCatalog.java), +[SupportsNamespaces](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/SupportsNamespaces.java), [ViewCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/ViewCatalog.java) classes. Right now, the plugin only provides support for Spark 3.5, Scala version 2.12 and 2.13, -and depends on iceberg-spark-runtime 1.8.1. +and depends on iceberg-spark-runtime 1.9.0. # Build Plugin Jar A task createPolarisSparkJar is added to build a jar for the Polaris Spark plugin, the jar is named as: "polaris-iceberg--spark-runtime-_.jar" -Building the Polaris project produces client jars for both Scala 2.12 and 2.13, and CI runs the Spark +Building the Polaris project produces client jars for both Scala 2.12 and 2.13, and CI runs the Spark client tests for both Scala versions as well. The Jar can also be built alone with a specific version using target `:polaris-spark-3.5_`. For example: diff --git a/quarkus/admin/distribution/LICENSE b/quarkus/admin/distribution/LICENSE index ddad4d567b..5256a16302 100644 --- a/quarkus/admin/distribution/LICENSE +++ b/quarkus/admin/distribution/LICENSE @@ -1344,43 +1344,43 @@ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-api Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-api Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-aws Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-aws Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-azure Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-azure Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-bundled-guava Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-bundled-guava Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-common Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-common Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-core Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-core Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-gcp Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-gcp Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt diff --git a/quarkus/server/distribution/LICENSE b/quarkus/server/distribution/LICENSE index 43e8850ec4..2fffdb7538 100644 --- a/quarkus/server/distribution/LICENSE +++ b/quarkus/server/distribution/LICENSE @@ -1966,43 +1966,43 @@ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-api Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-api Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-aws Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-aws Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-azure Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-azure Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-bundled-guava Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-bundled-guava Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-common Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-common Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-core Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-core Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt -------------------------------------------------------------------------------- -Group: org.apache.iceberg Name: iceberg-gcp Version: 1.8.1 +Group: org.apache.iceberg Name: iceberg-gcp Version: 1.9.0 Project URL (from POM): https://iceberg.apache.org/ License (from POM): Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt From 0d00e62425b10c93c271a2a0c69da95d78f6f111 Mon Sep 17 00:00:00 2001 From: Yong Date: Sat, 3 May 2025 22:30:55 -0500 Subject: [PATCH 2/3] Additional refs update for iceberg 1.9.0 --- getting-started/eclipselink/docker-compose.yml | 2 +- getting-started/jdbc/docker-compose.yml | 2 +- plugins/spark/README.md | 2 +- site/content/in-dev/unreleased/getting-started/using-polaris.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/getting-started/eclipselink/docker-compose.yml b/getting-started/eclipselink/docker-compose.yml index 0c2ca17a8b..5589dfb3ac 100644 --- a/getting-started/eclipselink/docker-compose.yml +++ b/getting-started/eclipselink/docker-compose.yml @@ -73,7 +73,7 @@ services: retries: 15 command: [ /opt/spark/bin/spark-sql, - --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0", + --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0", --conf, "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", --conf, "spark.sql.catalog.quickstart_catalog=org.apache.iceberg.spark.SparkCatalog", --conf, "spark.sql.catalog.quickstart_catalog.type=rest", diff --git a/getting-started/jdbc/docker-compose.yml b/getting-started/jdbc/docker-compose.yml index 4c85226c23..924f20daf3 100644 --- a/getting-started/jdbc/docker-compose.yml +++ b/getting-started/jdbc/docker-compose.yml @@ -73,7 +73,7 @@ services: retries: 15 command: [ /opt/spark/bin/spark-sql, - --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.7.0,org.apache.iceberg:iceberg-azure-bundle:1.7.0", + --packages, "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,software.amazon.awssdk:bundle:2.28.17,software.amazon.awssdk:url-connection-client:2.28.17,org.apache.iceberg:iceberg-gcp-bundle:1.9.0,org.apache.iceberg:iceberg-azure-bundle:1.9.0", --conf, "spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", --conf, "spark.sql.catalog.polaris=org.apache.iceberg.spark.SparkCatalog", --conf, "spark.sql.catalog.polaris.type=rest", diff --git a/plugins/spark/README.md b/plugins/spark/README.md index 5fc88d7344..07fa8ed862 100644 --- a/plugins/spark/README.md +++ b/plugins/spark/README.md @@ -26,7 +26,7 @@ REST endpoints, and provides implementations for Apache Spark's [ViewCatalog](https://github.com/apache/spark/blob/v3.5.5/sql/catalyst/src/main/java/org/apache/spark/sql/connector/catalog/ViewCatalog.java) classes. Right now, the plugin only provides support for Spark 3.5, Scala version 2.12 and 2.13, -and depends on iceberg-spark-runtime 1.8.1. +and depends on iceberg-spark-runtime 1.9.0. # Build Plugin Jar A task createPolarisSparkJar is added to build a jar for the Polaris Spark plugin, the jar is named as: diff --git a/site/content/in-dev/unreleased/getting-started/using-polaris.md b/site/content/in-dev/unreleased/getting-started/using-polaris.md index ff83e8748d..75004e3ea9 100644 --- a/site/content/in-dev/unreleased/getting-started/using-polaris.md +++ b/site/content/in-dev/unreleased/getting-started/using-polaris.md @@ -147,7 +147,7 @@ _Note: the credentials provided here are those for our principal, not the root c ```shell bin/spark-sql \ ---packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.7.1,org.apache.hadoop:hadoop-aws:3.4.0 \ +--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.9.0,org.apache.hadoop:hadoop-aws:3.4.0 \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ --conf spark.sql.catalog.quickstart_catalog.warehouse=quickstart_catalog \ --conf spark.sql.catalog.quickstart_catalog.header.X-Iceberg-Access-Delegation=vended-credentials \ From 12af93f5fcb9d689e68d7008ddd57d33c6cde2c4 Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 4 May 2025 01:03:41 -0500 Subject: [PATCH 3/3] Additional refs update for iceberg 1.9.0 --- plugins/spark/v3.5/spark/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index 5ce7e73c05..600d150f4f 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -167,9 +167,11 @@ tasks.register("createPolarisSparkJar") { minimize { exclude(dependency("org.apache.iceberg:iceberg-spark-runtime-*.*")) exclude(dependency("org.apache.iceberg:iceberg-core*.*")) + exclude(dependency("org.apache.avro:avro*.*")) } relocate("com.fasterxml", "org.apache.polaris.shaded.com.fasterxml.jackson") + relocate("org.apache.avro", "org.apache.polaris.shaded.org.apache.avro") } tasks.withType(Jar::class).named("sourcesJar") { dependsOn("createPolarisSparkJar") }