From f5ad230ce51bc61d972142db10d7b58867f04821 Mon Sep 17 00:00:00 2001 From: "Qian.Sun" Date: Sat, 2 Apr 2022 13:29:08 +0800 Subject: [PATCH 1/2] [SPARK-38770][K8S] Simply steps to re write primary resource in k8s spark application --- .../deploy/k8s/features/DriverCommandFeatureStep.scala | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala index 8015a1af3e17d..5c2371cbfe4b2 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala @@ -67,7 +67,7 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf) // re-write primary resource, app jar is also added to spark.jars by default in SparkSubmit // no uploading takes place here val newResName = KubernetesUtils - .renameMainAppResource(resource = res, shouldUploadLocal = false) + .renameMainAppResource(res, Option(conf.sparkConf), false) val driverContainer = baseDriverContainer(pod, newResName).build() SparkPod(pod.pod, driverContainer) } @@ -120,12 +120,6 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf) } private def baseDriverContainer(pod: SparkPod, resource: String): ContainerBuilder = { - // re-write primary resource, app jar is also added to spark.jars by default in SparkSubmit - val resolvedResource = if (conf.mainAppResource.isInstanceOf[JavaMainAppResource]) { - KubernetesUtils.renameMainAppResource(resource, Option(conf.sparkConf), false) - } else { - resource - } var proxyUserArgs = Seq[String]() if (!conf.proxyUser.isEmpty) { proxyUserArgs = proxyUserArgs :+ "--proxy-user" @@ -136,7 +130,7 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf) .addToArgs(proxyUserArgs: _*) .addToArgs("--properties-file", SPARK_CONF_PATH) .addToArgs("--class", conf.mainClass) - .addToArgs(resolvedResource) + .addToArgs(resource) .addToArgs(conf.appArgs: _*) } } From 7dd1d0517b21ab277fcd2db785b3f2ce397c03c3 Mon Sep 17 00:00:00 2001 From: "Qian.Sun" Date: Sun, 3 Apr 2022 08:15:25 +0800 Subject: [PATCH 2/2] address comments --- .../spark/deploy/k8s/features/DriverCommandFeatureStep.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala index 5c2371cbfe4b2..e9fbcd795cc00 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/DriverCommandFeatureStep.scala @@ -67,7 +67,7 @@ private[spark] class DriverCommandFeatureStep(conf: KubernetesDriverConf) // re-write primary resource, app jar is also added to spark.jars by default in SparkSubmit // no uploading takes place here val newResName = KubernetesUtils - .renameMainAppResource(res, Option(conf.sparkConf), false) + .renameMainAppResource(resource = res, shouldUploadLocal = false) val driverContainer = baseDriverContainer(pod, newResName).build() SparkPod(pod.pod, driverContainer) }