From 69c0d2711f54f527db1e5b1fa28c9c14353ddff9 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 10 Mar 2025 11:50:52 +0800 Subject: [PATCH 1/2] [SPARK-51449][BUILD] Restore hive-llap-common to compile scope --- LICENSE-binary | 1 + NOTICE-binary | 3 +++ dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 + pom.xml | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/LICENSE-binary b/LICENSE-binary index 1fc531d800522..8d744385ae9ff 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -332,6 +332,7 @@ org.apache.hive:hive-cli org.apache.hive:hive-common org.apache.hive:hive-exec org.apache.hive:hive-jdbc +org.apache.hive:hive-llap-common org.apache.hive:hive-metastore org.apache.hive:hive-serde org.apache.hive:hive-service-rpc diff --git a/NOTICE-binary b/NOTICE-binary index a3f302b1cb04d..172d58ed657b7 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -1047,6 +1047,9 @@ Copyright 2019 The Apache Software Foundation Hive Query Language Copyright 2019 The Apache Software Foundation +Hive Llap Common +Copyright 2019 The Apache Software Foundation + Hive Metastore Copyright 2019 The Apache Software Foundation diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 464a197236974..328296b68413b 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -86,6 +86,7 @@ hive-cli/2.3.10//hive-cli-2.3.10.jar hive-common/2.3.10//hive-common-2.3.10.jar hive-exec/2.3.10/core/hive-exec-2.3.10-core.jar hive-jdbc/2.3.10//hive-jdbc-2.3.10.jar +hive-llap-common/2.3.10//hive-llap-common-2.3.10.jar hive-metastore/2.3.10//hive-metastore-2.3.10.jar hive-serde/2.3.10//hive-serde-2.3.10.jar hive-service-rpc/4.0.0//hive-service-rpc-4.0.0.jar diff --git a/pom.xml b/pom.xml index 0670d9f8d3887..d58550c5642a9 100644 --- a/pom.xml +++ b/pom.xml @@ -275,7 +275,7 @@ compile compile compile - test + compile compile compile compile From 70618d62b8a2ebc4bd710f0fe66c20408ddf4814 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 10 Mar 2025 13:15:13 +0800 Subject: [PATCH 2/2] docs --- docs/sql-migration-guide.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index f459a88d8e148..254c54a414a7e 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -31,7 +31,6 @@ license: | - Since Spark 4.0, any read of SQL tables takes into consideration the SQL configs `spark.sql.files.ignoreCorruptFiles`/`spark.sql.files.ignoreMissingFiles` instead of the core config `spark.files.ignoreCorruptFiles`/`spark.files.ignoreMissingFiles`. - Since Spark 4.0, when reading SQL tables hits `org.apache.hadoop.security.AccessControlException` and `org.apache.hadoop.hdfs.BlockMissingException`, the exception will be thrown and fail the task, even if `spark.sql.files.ignoreCorruptFiles` is set to `true`. - Since Spark 4.0, `spark.sql.hive.metastore` drops the support of Hive prior to 2.0.0 as they require JDK 8 that Spark does not support anymore. Users should migrate to higher versions. -- Since Spark 4.0, Spark removes `hive-llap-common` dependency. To restore the previous behavior, add `hive-llap-common` jar to the class path. - Since Spark 4.0, `spark.sql.parquet.compression.codec` drops the support of codec name `lz4raw`, please use `lz4_raw` instead. - Since Spark 4.0, when overflowing during casting timestamp to byte/short/int under non-ansi mode, Spark will return null instead a wrapping value. - Since Spark 4.0, the `encode()` and `decode()` functions support only the following charsets 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'. To restore the previous behavior when the function accepts charsets of the current JDK used by Spark, set `spark.sql.legacy.javaCharsets` to `true`.