From e9d77838ae6106e592f035701ab444dce3023702 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Tue, 19 Mar 2024 07:49:30 -0700 Subject: [PATCH] [SPARK-47457][SQL] Fix `IsolatedClientLoader.supportsHadoopShadedClient` to handle Hadoop 3.4+ ### What changes were proposed in this pull request? This PR aims to fix `IsolatedClientLoader.supportsHadoopShadedClient` to handle Hadoop 3.4+ correctly. ### Why are the changes needed? Apache Spark 3.4+ support shaded clients, but currently `supportsHadoopShadedClient` returns `false`. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual review. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45585 from dongjoon-hyun/SPARK-47457. Authored-by: Dongjoon Hyun Signed-off-by: Dongjoon Hyun --- .../apache/spark/sql/hive/client/IsolatedClientLoader.scala | 2 ++ .../apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 18090b53e3c1..c80688912c5a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -113,6 +113,8 @@ private[hive] object IsolatedClientLoader extends Logging { VersionUtils.majorMinorPatchVersion(hadoopVersion).exists { case (3, 2, v) if v >= 2 => true case (3, 3, v) if v >= 1 => true + case (3, v, _) if v >= 4 => true + case (v, _, _) if v >= 4 => true case _ => false } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala index 2a921c3fd85b..a90fe4493dee 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HadoopVersionInfoSuite.scala @@ -72,12 +72,12 @@ class HadoopVersionInfoSuite extends SparkFunSuite { } test("SPARK-32212: test supportHadoopShadedClient()") { - Seq("3.2.2", "3.2.3", "3.2.2.1", "3.2.2-XYZ", "3.2.2.4-SNAPSHOT").foreach { version => + Seq("4", "3.2.2", "3.2.3", "3.2.2.1", "3.2.2-XYZ", "3.2.2.4-SNAPSHOT").foreach { version => assert(IsolatedClientLoader.supportsHadoopShadedClient(version), s"version $version") } // negative cases - Seq("3.1.3", "3.2", "3.2.1", "4").foreach { version => + Seq("3.1.3", "3.2", "3.2.1").foreach { version => assert(!IsolatedClientLoader.supportsHadoopShadedClient(version), s"version $version") } }