Skip to content

Commit 5e733ae

Browse files
author
pgandhi
committed
[SPARK-22151] : Made changes as per comments
- spark.executorEnv should not be overridden by appMasterEnv - Replacing ++ with ++=:
1 parent 0aee8fa commit 5e733ae

File tree

1 file changed

+4
-2
lines changed
  • resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn

1 file changed

+4
-2
lines changed

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -811,7 +811,7 @@ private[spark] class Client(
811811

812812
// Finally, update the Spark config to propagate PYTHONPATH to the AM and executors.
813813
if (pythonPath.nonEmpty) {
814-
val pythonPathStr = (sys.env.get("PYTHONPATH") ++ pythonPath)
814+
val pythonPathStr = (sys.env.get("PYTHONPATH") ++=: pythonPath)
815815
.mkString(ApplicationConstants.CLASS_PATH_SEPARATOR)
816816
val newValue =
817817
if (env.contains("PYTHONPATH")) {
@@ -820,7 +820,9 @@ private[spark] class Client(
820820
pythonPathStr
821821
}
822822
env("PYTHONPATH") = newValue
823-
sparkConf.setExecutorEnv("PYTHONPATH", newValue)
823+
if (!sparkConf.getExecutorEnv.toMap.contains("PYTHONPATH")) {
824+
sparkConf.setExecutorEnv("PYTHONPATH", newValue)
825+
}
824826
}
825827

826828
if (isClusterMode) {

0 commit comments

Comments
 (0)