Skip to content

Commit d152f9f

Browse files
committed
Address the comments
1 parent f9b62a1 commit d152f9f

File tree

3 files changed

+6
-7
lines changed

3 files changed

+6
-7
lines changed

core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,7 +616,8 @@ object SparkSubmit {
616616
"spark.jars",
617617
"spark.files",
618618
"spark.yarn.dist.files",
619-
"spark.yarn.dist.archives")
619+
"spark.yarn.dist.archives",
620+
"spark.yarn.dist.jars")
620621
pathConfigs.foreach { config =>
621622
// Replace old URIs with resolved URIs, if they exist
622623
sysProps.get(config).foreach { oldValue =>

core/src/main/scala/org/apache/spark/internal/config/package.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ package object config {
3434
private[spark] val DRIVER_USER_CLASS_PATH_FIRST =
3535
ConfigBuilder("spark.driver.userClassPathFirst").booleanConf.withDefault(false)
3636

37-
private[spark] val DRIVER_MEMORY = ConfigBuilder("spark.driver.memory")
37+
private[spark] val DRIVER_MEMORY = ConfigBuilder("spark.driver.memory")
3838
.bytesConf(ByteUnit.MiB)
3939
.withDefaultString("1g")
4040

yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ private[spark] class Client(
498498
* Copy user jar to the distributed cache if their scheme is not "local".
499499
* Otherwise, set the corresponding key in our SparkConf to handle it downstream.
500500
*/
501-
for (jar <- Option(args.userJar); if !jar.trim.isEmpty) {
501+
Option(args.userJar).filter(_.trim.nonEmpty).foreach { jar =>
502502
val (isLocal, localizedPath) = distribute(jar, destName = Some(APP_JAR_NAME))
503503
if (isLocal) {
504504
require(localizedPath != null, s"Path $jar already distributed")
@@ -516,12 +516,10 @@ private[spark] class Client(
516516
* (3) whether to add these resources to the classpath
517517
*/
518518
val cachedSecondaryJarLinks = ListBuffer.empty[String]
519-
val files = sparkConf.get(FILES_TO_DISTRIBUTE).map(p => Utils.resolveURIs(p)).orNull
520-
val archives = sparkConf.get(ARCHIVES_TO_DISTRIBUTE).map(p => Utils.resolveURIs(p)).orNull
521519
List(
522520
(sparkConf.get(JARS_TO_DISTRIBUTE).orNull, LocalResourceType.FILE, true),
523-
(files, LocalResourceType.FILE, false),
524-
(archives, LocalResourceType.ARCHIVE, false)
521+
(sparkConf.get(FILES_TO_DISTRIBUTE).orNull, LocalResourceType.FILE, false),
522+
(sparkConf.get(ARCHIVES_TO_DISTRIBUTE).orNull, LocalResourceType.ARCHIVE, false)
525523
).foreach { case (flist, resType, addToClasspath) =>
526524
if (flist != null && !flist.isEmpty()) {
527525
flist.split(',').foreach { file =>

0 commit comments

Comments
 (0)