Skip to content

Commit b2e377f

Browse files
author
Marcelo Vanzin
committed
Review feedback.
1 parent 93c3f85 commit b2e377f

File tree

2 files changed

+21
-11
lines changed

2 files changed

+21
-11
lines changed

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -395,7 +395,15 @@ object ClientBase extends Logging {
395395
val APP_JAR: String = "__app__.jar"
396396
val LOCAL_SCHEME = "local"
397397
val CONF_SPARK_JAR = "spark.yarn.jar"
398+
/**
399+
* This is an internal config used to propagate the location of the user's jar file to the
400+
* driver/executors.
401+
*/
398402
val CONF_SPARK_USER_JAR = "spark.yarn.user.jar"
403+
/**
404+
* This is an internal config used to propagate the list of extra jars to add to the classpath
405+
* of executors.
406+
*/
399407
val CONF_SPARK_YARN_SECONDARY_JARS = "spark.yarn.secondary.jars"
400408
val ENV_SPARK_JAR = "SPARK_JAR"
401409

@@ -506,8 +514,8 @@ object ClientBase extends Logging {
506514
addUserClasspath(args, sparkConf, env)
507515
}
508516

509-
// Append all class files and jar files under the working directory to the classpath.
510-
addFileToClasspath("*", null, env)
517+
// Append all jar files under the working directory to the classpath.
518+
addClasspathEntry(Environment.PWD.$() + Path.SEPARATOR + "*", env);
511519
}
512520

513521
/**

yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
3232
import org.mockito.Matchers._
3333
import org.mockito.Mockito._
3434
import org.scalatest.FunSuite
35-
import org.scalatest.matchers.ShouldMatchers._
35+
import org.scalatest.Matchers
3636

3737
import scala.collection.JavaConversions._
3838
import scala.collection.mutable.{ HashMap => MutableHashMap }
@@ -41,7 +41,7 @@ import scala.util.Try
4141
import org.apache.spark.SparkConf
4242
import org.apache.spark.util.Utils
4343

44-
class ClientBaseSuite extends FunSuite {
44+
class ClientBaseSuite extends FunSuite with Matchers {
4545

4646
test("default Yarn application classpath") {
4747
ClientBase.getDefaultYarnApplicationClasspath should be(Some(Fixtures.knownDefYarnAppCP))
@@ -89,17 +89,19 @@ class ClientBaseSuite extends FunSuite {
8989

9090
ClientBase.populateClasspath(args, conf, sparkConf, env, None)
9191

92-
val jars = env("CLASSPATH").split(File.pathSeparator)
93-
s"$SPARK,$USER,$ADDED".split(",").foreach({ jar =>
94-
val uri = new URI(jar)
92+
val cp = env("CLASSPATH").split(File.pathSeparator)
93+
s"$SPARK,$USER,$ADDED".split(",").foreach({ entry =>
94+
val uri = new URI(entry)
9595
if (ClientBase.LOCAL_SCHEME.equals(uri.getScheme())) {
96-
jars should contain (uri.getPath())
96+
cp should contain (uri.getPath())
9797
} else {
98-
jars should not contain (uri.getPath())
98+
cp should not contain (uri.getPath())
9999
}
100100
})
101-
jars should not contain (ClientBase.SPARK_JAR)
102-
jars should not contain (ClientBase.APP_JAR)
101+
cp should contain (s"$$PWD")
102+
cp should contain (s"$$PWD${File.separator}*")
103+
cp should not contain (ClientBase.SPARK_JAR)
104+
cp should not contain (ClientBase.APP_JAR)
103105
}
104106

105107
test("Jar path propagation through SparkConf") {

0 commit comments

Comments
 (0)