[SPARK-44914][BUILD] Upgrade Apache Ivy to 2.5.2

Author: dongjoon-hyun

common/utils/src/main/scala/org/apache/spark/util/MavenUtils.scala

@@ -324,6 +324,13 @@ private[spark] object MavenUtils extends Logging {
     val ivySettings: IvySettings = new IvySettings
     try {
       ivySettings.load(file)
+      if (ivySettings.getDefaultIvyUserDir == null && ivySettings.getDefaultCache == null) {
+        // To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility.
+        // `processIvyPathArg` can overwrite these later.
+        val alternateIvyDir = System.getProperty("user.home") + File.separator + ".ivy2.5.2"
+        ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir))
+        ivySettings.setDefaultCache(new File(alternateIvyDir, "cache"))
+      }
     } catch {
       case e @ (_: IOException | _: ParseException) =>
         throw new SparkException(s"Failed when loading Ivy settings from $settingsFile", e)
@@ -335,10 +342,12 @@ private[spark] object MavenUtils extends Logging {
 
   /* Set ivy settings for location of cache, if option is supplied */
   private def processIvyPathArg(ivySettings: IvySettings, ivyPath: Option[String]): Unit = {
-    ivyPath.filterNot(_.trim.isEmpty).foreach { alternateIvyDir =>
-      ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir))
-      ivySettings.setDefaultCache(new File(alternateIvyDir, "cache"))
+    val alternateIvyDir = ivyPath.filterNot(_.trim.isEmpty).getOrElse {
+      // To protect old Ivy-based systems like old Spark from Apache Ivy 2.5.2's incompatibility.
+      System.getProperty("user.home") + File.separator + ".ivy2.5.2"
     }
+    ivySettings.setDefaultIvyUserDir(new File(alternateIvyDir))
+    ivySettings.setDefaultCache(new File(alternateIvyDir, "cache"))
   }
 
   /* Add any optional additional remote repositories */

common/utils/src/test/scala/org/apache/spark/util/IvyTestUtils.scala

@@ -374,7 +374,8 @@ private[spark] object IvyTestUtils {
       f(repo.toURI.toString)
     } finally {
       // Clean up
-      if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2")) {
+      if (repo.toString.contains(".m2") || repo.toString.contains(".ivy2") ||
+          repo.toString.contains(".ivy2.5.2")) {
         val groupDir = getBaseGroupDirectory(artifact, useIvyLayout)
         FileUtils.deleteDirectory(new File(repo, groupDir + File.separator + artifact.artifactId))
         deps.foreach { _.foreach { dep =>

core/src/main/scala/org/apache/spark/internal/config/package.scala

@@ -2491,10 +2491,10 @@ package object config {
       .doc("Path to specify the Ivy user directory, used for the local Ivy cache and " +
         "package files from spark.jars.packages. " +
         "This will override the Ivy property ivy.default.ivy.user.dir " +
-        "which defaults to ~/.ivy2.")
+        "which defaults to ~/.ivy2.5.2")
       .version("1.3.0")
       .stringConf
-      .createOptional
+      .createWithDefault("~/.ivy2.5.2")
 
   private[spark] val JAR_IVY_SETTING_PATH =
     ConfigBuilder(MavenUtils.JAR_IVY_SETTING_PATH_KEY)

dev/deps/spark-deps-hadoop-3-hive-2.3

@@ -102,7 +102,7 @@ httpcore/4.4.16//httpcore-4.4.16.jar
 icu4j/72.1//icu4j-72.1.jar
 ini4j/0.5.4//ini4j-0.5.4.jar
 istack-commons-runtime/3.0.8//istack-commons-runtime-3.0.8.jar
-ivy/2.5.1//ivy-2.5.1.jar
+ivy/2.5.2//ivy-2.5.2.jar
 jackson-annotations/2.16.1//jackson-annotations-2.16.1.jar
 jackson-core-asl/1.9.13//jackson-core-asl-1.9.13.jar
 jackson-core/2.16.1//jackson-core-2.16.1.jar

dev/run-tests.py

@@ -478,6 +478,8 @@ def main():
     rm_r(os.path.join(SPARK_HOME, "work"))
     rm_r(os.path.join(USER_HOME, ".ivy2", "local", "org.apache.spark"))
     rm_r(os.path.join(USER_HOME, ".ivy2", "cache", "org.apache.spark"))
+    rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "local", "org.apache.spark"))
+    rm_r(os.path.join(USER_HOME, ".ivy2.5.2", "cache", "org.apache.spark"))
 
     os.environ["CURRENT_BLOCK"] = str(ERROR_CODES["BLOCK_GENERAL"])
 

docs/core-migration-guide.md

@@ -36,6 +36,8 @@ license: |
 
 - Since Spark 4.0, Spark uses `ReadWriteOncePod` instead of `ReadWriteOnce` access mode in persistence volume claims. To restore the legacy behavior, you can set `spark.kubernetes.legacy.useReadWriteOnceAccessMode` to `true`.
 
+- Since Spark 4.0, Spark uses `~/.ivy2.5.2` as Ivy user directory by default to isolate the existing systems from Apache Ivy's incompatibility. To restore the legacy behavior, you can set `spark.jars.ivy` to `~/.ivy2`.
+
 ## Upgrading from Core 3.4 to 3.5
 
 - Since Spark 3.5, `spark.yarn.executor.failuresValidityInterval` is deprecated. Use `spark.executor.failuresValidityInterval` instead.

pom.xml

@@ -146,11 +146,7 @@
     <jetty.version>10.0.19</jetty.version>
     <jakartaservlet.version>4.0.3</jakartaservlet.version>
     <chill.version>0.10.0</chill.version>
-    <!--
-      SPARK-44968: don't upgrade Ivy to version 2.5.2 until the test aborted of
-      `HiveExternalCatalogVersionsSuite` in Java 11/17 daily tests is resolved.
-    -->
-    <ivy.version>2.5.1</ivy.version>
+    <ivy.version>2.5.2</ivy.version>
     <oro.version>2.0.8</oro.version>
     <!--
     If you change codahale.metrics.version, you also need to change

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

@@ -58,6 +58,7 @@ import org.apache.spark.sql.types._
 import org.apache.spark.tags.ExtendedSQLTest
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
 import org.apache.spark.util.ResetSystemProperties
+import org.apache.spark.util.Utils
 
 @ExtendedSQLTest
 class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSparkPlanHelper
@@ -3873,12 +3874,12 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
   test("SPARK-33084: Add jar support Ivy URI in SQL -- jar contains udf class") {
     val sumFuncClass = "org.apache.spark.examples.sql.Spark33084"
     val functionName = "test_udf"
-    withTempDir { dir =>
-      System.setProperty("ivy.home", dir.getAbsolutePath)
+    val targetCacheJarDir = new File(
+      System.getProperty("user.home") + File.separator + ".ivy2.5.2",
+      "/local/org.apache.spark/SPARK-33084/1.0/jars/")
+    try {
       val sourceJar = new File(Thread.currentThread().getContextClassLoader
         .getResource("SPARK-33084.jar").getFile)
-      val targetCacheJarDir = new File(dir.getAbsolutePath +
-        "/local/org.apache.spark/SPARK-33084/1.0/jars/")
       targetCacheJarDir.mkdir()
       // copy jar to local cache
       FileUtils.copyFileToDirectory(sourceJar, targetCacheJarDir)
@@ -3905,6 +3906,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
           checkAnswer(sql("SELECT * FROM v1"), Seq(Row(2.0)))
         }
       }
+    } finally {
+      Utils.deleteRecursively(targetCacheJarDir)
     }
   }