File tree Expand file tree Collapse file tree 1 file changed +4
-15
lines changed
resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn Expand file tree Collapse file tree 1 file changed +4
-15
lines changed Original file line number Diff line number Diff line change @@ -811,22 +811,11 @@ private[spark] class Client(
811811
812812 // Finally, update the Spark config to propagate PYTHONPATH to the AM and executors.
813813 if (pythonPath.nonEmpty) {
814- val pythonPathStr = (sys.env.get(" PYTHONPATH" ) ++ pythonPath)
814+ val pythonPathList = (sys.env.get(" PYTHONPATH" ) ++ pythonPath)
815+ env(" PYTHONPATH" ) = (env.get(" PYTHONPATH" ) ++ pythonPathList)
815816 .mkString(ApplicationConstants .CLASS_PATH_SEPARATOR )
816- val newValue =
817- if (env.contains(" PYTHONPATH" )) {
818- env(" PYTHONPATH" ) + ApplicationConstants .CLASS_PATH_SEPARATOR + pythonPathStr
819- } else {
820- pythonPathStr
821- }
822- env(" PYTHONPATH" ) = newValue
823- val pythonPathExecutorEnv =
824- if (! sparkConf.getExecutorEnv.toMap.contains(" PYTHONPATH" )) {
825- pythonPathStr
826- } else {
827- sparkConf.getExecutorEnv.toMap.get(" PYTHONPATH" ).get +
828- ApplicationConstants .CLASS_PATH_SEPARATOR + pythonPathStr
829- }
817+ val pythonPathExecutorEnv = (sparkConf.getExecutorEnv.toMap.get(" PYTHONPATH" ) ++
818+ pythonPathList).mkString(ApplicationConstants .CLASS_PATH_SEPARATOR )
830819 sparkConf.setExecutorEnv(" PYTHONPATH" , pythonPathExecutorEnv)
831820 }
832821
You can’t perform that action at this time.
0 commit comments