From d21927d14798dc576eb34080ae9afe00fa926c69 Mon Sep 17 00:00:00 2001 From: tianlzhang Date: Wed, 25 May 2022 09:41:40 +0800 Subject: [PATCH] [SPARK-39278][CORE] Fix backward compatibility of alternative configs of Hadoop Filesystems to access --- core/src/main/scala/org/apache/spark/SparkConf.scala | 4 ++-- core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala index 00a0f61ab47b..02492f947024 100644 --- a/core/src/main/scala/org/apache/spark/SparkConf.scala +++ b/core/src/main/scala/org/apache/spark/SparkConf.scala @@ -714,8 +714,8 @@ private[spark] object SparkConf extends Logging { KERBEROS_RELOGIN_PERIOD.key -> Seq( AlternateConfig("spark.yarn.kerberos.relogin.period", "3.0")), KERBEROS_FILESYSTEMS_TO_ACCESS.key -> Seq( - AlternateConfig("spark.yarn.access.namenodes", "2.2"), - AlternateConfig("spark.yarn.access.hadoopFileSystems", "3.0")), + AlternateConfig("spark.yarn.access.hadoopFileSystems", "3.0"), + AlternateConfig("spark.yarn.access.namenodes", "2.2")), "spark.kafka.consumer.cache.capacity" -> Seq( AlternateConfig("spark.sql.kafkaConsumerCache.capacity", "3.0")) ) diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala index 7ae1eac1db18..e5a3d4216db2 100644 --- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala +++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala @@ -277,8 +277,11 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst conf.set("spark.yarn.access.namenodes", "testNode") assert(conf.get(KERBEROS_FILESYSTEMS_TO_ACCESS) === Array("testNode")) - conf.set("spark.yarn.access.hadoopFileSystems", "testNode") - assert(conf.get(KERBEROS_FILESYSTEMS_TO_ACCESS) === Array("testNode")) + conf.set("spark.yarn.access.hadoopFileSystems", "testNode2") + assert(conf.get(KERBEROS_FILESYSTEMS_TO_ACCESS) === Array("testNode2")) + + conf.set("spark.yarn.access.namenodes", "testNode3") + assert(conf.get(KERBEROS_FILESYSTEMS_TO_ACCESS) === Array("testNode2")) } test("akka deprecated configs") {