From a1fcf7fa9a7c06640902f8b113a622679b2394b4 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Wed, 19 Oct 2022 18:02:21 +0800 Subject: [PATCH 1/3] fix --- .../spark/SparkBatchProcessBuilder.scala | 1 + .../engine/spark/SparkProcessBuilder.scala | 27 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala index 89ce8418077..e7de6baa449 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkBatchProcessBuilder.scala @@ -54,6 +54,7 @@ class SparkBatchProcessBuilder( buffer += s"${convertConfigKey(k)}=$v" } + setSparkUserName(proxyUser, buffer) buffer += PROXY_USER buffer += proxyUser diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 052887c927b..c933de796cf 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -91,26 +91,14 @@ class SparkProcessBuilder( buffer += s"${convertConfigKey(k)}=$v" } - // For spark on kubernetes, spark pod using env SPARK_USER_NAME as current user - def setSparkUserName(userName: String): Unit = { - clusterManager().foreach(cm => { - if (cm.startsWith("k8s://")) { - buffer += CONF - buffer += s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$userName" - buffer += CONF - buffer += s"spark.executorEnv.SPARK_USER_NAME=$userName" - } - }) - } - // if the keytab is specified, PROXY_USER is not supported tryKeytab() match { case None => - setSparkUserName(proxyUser) + setSparkUserName(proxyUser, buffer) buffer += PROXY_USER buffer += proxyUser case Some(name) => - setSparkUserName(name) + setSparkUserName(name, buffer) } mainResource.foreach { r => buffer += r } @@ -185,6 +173,17 @@ class SparkProcessBuilder( override def validateConf: Unit = Validator.validateConf(conf) + // For spark on kubernetes, spark pod using env SPARK_USER_NAME as current user + def setSparkUserName(userName: String, buffer: ArrayBuffer[String]): Unit = { + clusterManager().foreach(cm => { + if (cm.startsWith("k8s://")) { + buffer += CONF + buffer += s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$userName" + buffer += CONF + buffer += s"spark.executorEnv.SPARK_USER_NAME=$userName" + } + }) + } } object SparkProcessBuilder { From 0d405082ee4ac76a8ad08e6af6afc32d1e7a3f31 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 20 Oct 2022 10:23:00 +0800 Subject: [PATCH 2/3] fix --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index c933de796cf..214dd9b533c 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -176,7 +176,7 @@ class SparkProcessBuilder( // For spark on kubernetes, spark pod using env SPARK_USER_NAME as current user def setSparkUserName(userName: String, buffer: ArrayBuffer[String]): Unit = { clusterManager().foreach(cm => { - if (cm.startsWith("k8s://")) { + if (cm.toUpperCase.startsWith("K8S")) { buffer += CONF buffer += s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$userName" buffer += CONF From 76dee3e3f64c3a3bd436d10e9c321c5709bb8db6 Mon Sep 17 00:00:00 2001 From: zwangsheng <2213335496@qq.com> Date: Thu, 20 Oct 2022 10:52:32 +0800 Subject: [PATCH 3/3] style --- .../org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala index 214dd9b533c..cd12dbd36c4 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala +++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala @@ -175,14 +175,14 @@ class SparkProcessBuilder( // For spark on kubernetes, spark pod using env SPARK_USER_NAME as current user def setSparkUserName(userName: String, buffer: ArrayBuffer[String]): Unit = { - clusterManager().foreach(cm => { + clusterManager().foreach { cm => if (cm.toUpperCase.startsWith("K8S")) { buffer += CONF buffer += s"spark.kubernetes.driverEnv.SPARK_USER_NAME=$userName" buffer += CONF buffer += s"spark.executorEnv.SPARK_USER_NAME=$userName" } - }) + } } }