Skip to content

[SPARK-10901] [YARN] spark.yarn.user.classpath.first doesn't work #8959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

[SPARK-10901] [YARN] spark.yarn.user.classpath.first doesn't work #8959

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
08f9382
[SPARK-10901] spark.yarn.user.classpath.first doesn't work
Oct 1, 2015
b860226
Rework
Oct 5, 2015
cd1bae0
combine getUserClasspath functions
Oct 5, 2015
e627adc
fix scalastyle
Oct 5, 2015
ac3ffcf
scalastyle again
Oct 5, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 27 additions & 12 deletions yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1153,13 +1153,24 @@ object Client extends Logging {
}

if (sparkConf.getBoolean("spark.yarn.user.classpath.first", false)) {
val userClassPath =
// in order to properly add the app jar when user classpath is first
// we have to do the mainJar separate in order to send the right thing
// into addFileToClasspath
val mainJar =
if (args != null) {
getUserClasspath(Option(args.userJar), Option(args.addJars))
getMainJarUri(Option(args.userJar))
} else {
getUserClasspath(sparkConf)
getMainJarUri(sparkConf.getOption(CONF_SPARK_USER_JAR))
}
userClassPath.foreach { x =>
mainJar.foreach(addFileToClasspath(sparkConf, _, APP_JAR, env))

val secondaryJars =
if (args != null) {
getSecondaryJarUris(Option(args.addJars))
} else {
getSecondaryJarUris(sparkConf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
}
secondaryJars.foreach { x =>
addFileToClasspath(sparkConf, x, null, env)
}
}
Expand All @@ -1176,16 +1187,20 @@ object Client extends Logging {
* @param conf Spark configuration.
*/
def getUserClasspath(conf: SparkConf): Array[URI] = {
getUserClasspath(conf.getOption(CONF_SPARK_USER_JAR),
conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
val mainUri = getMainJarUri(conf.getOption(CONF_SPARK_USER_JAR))
val secondaryUris = getSecondaryJarUris(conf.getOption(CONF_SPARK_YARN_SECONDARY_JARS))
(mainUri ++ secondaryUris).toArray
}

private def getUserClasspath(
mainJar: Option[String],
secondaryJars: Option[String]): Array[URI] = {
val mainUri = mainJar.orElse(Some(APP_JAR)).map(new URI(_))
val secondaryUris = secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_))
(mainUri ++ secondaryUris).toArray
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
mainJar.flatMap { path =>
val uri = new URI(path)
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
}.orElse(Some(new URI(APP_JAR)))
}

private def getSecondaryJarUris(secondaryJars: Option[String]): Seq[URI] = {
secondaryJars.map(_.split(",")).toSeq.flatten.map(new URI(_))
}

/**
Expand Down