Skip to content

Commit cd55fbc

Browse files
author
Marcelo Vanzin
committed
[SPARK-11023] [YARN] Avoid creating URIs from local paths directly.
The issue is that local paths on Windows, when provided with drive letters or backslashes, are not valid URIs. Instead of trying to figure out whether paths are URIs or not, use Utils.resolveURI() which does that for us. Author: Marcelo Vanzin <[email protected]> Closes #9049 from vanzin/SPARK-11023 and squashes the following commits: 77021f2 [Marcelo Vanzin] [SPARK-11023] [yarn] Avoid creating URIs from local paths directly. (cherry picked from commit 149472a)
1 parent 6dc23e6 commit cd55fbc

File tree

1 file changed

+6
-5
lines changed
  • yarn/src/main/scala/org/apache/spark/deploy/yarn

1 file changed

+6
-5
lines changed

yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,8 @@ private[spark] class Client(
318318
destName: Option[String] = None,
319319
targetDir: Option[String] = None,
320320
appMasterOnly: Boolean = false): (Boolean, String) = {
321-
val localURI = new URI(path.trim())
321+
val trimmedPath = path.trim()
322+
val localURI = Utils.resolveURI(trimmedPath)
322323
if (localURI.getScheme != LOCAL_SCHEME) {
323324
if (addDistributedUri(localURI)) {
324325
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
@@ -334,7 +335,7 @@ private[spark] class Client(
334335
(false, null)
335336
}
336337
} else {
337-
(true, path.trim())
338+
(true, trimmedPath)
338339
}
339340
}
340341

@@ -555,10 +556,10 @@ private[spark] class Client(
555556
LOCALIZED_PYTHON_DIR)
556557
}
557558
(pySparkArchives ++ pyArchives).foreach { path =>
558-
val uri = new URI(path)
559+
val uri = Utils.resolveURI(path)
559560
if (uri.getScheme != LOCAL_SCHEME) {
560561
pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD),
561-
new Path(path).getName())
562+
new Path(uri).getName())
562563
} else {
563564
pythonPath += uri.getPath()
564565
}
@@ -1175,7 +1176,7 @@ object Client extends Logging {
11751176

11761177
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
11771178
mainJar.flatMap { path =>
1178-
val uri = new URI(path)
1179+
val uri = Utils.resolveURI(path)
11791180
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
11801181
}.orElse(Some(new URI(APP_JAR)))
11811182
}

0 commit comments

Comments
 (0)