Skip to content

Commit 77021f2

Browse files
author
Marcelo Vanzin
committed
[SPARK-11023] [yarn] Avoid creating URIs from local paths directly.
The issue is that local paths on Windows, when provided with drive letters or backslashes, are not valid URIs. Instead of trying to figure out whether paths are URIs or not, use Utils.resolveURI() which does that for us.
1 parent 70f44ad commit 77021f2

File tree

1 file changed

+6
-5
lines changed
  • yarn/src/main/scala/org/apache/spark/deploy/yarn

1 file changed

+6
-5
lines changed

yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,8 @@ private[spark] class Client(
340340
destName: Option[String] = None,
341341
targetDir: Option[String] = None,
342342
appMasterOnly: Boolean = false): (Boolean, String) = {
343-
val localURI = new URI(path.trim())
343+
val trimmedPath = path.trim()
344+
val localURI = Utils.resolveURI(trimmedPath)
344345
if (localURI.getScheme != LOCAL_SCHEME) {
345346
if (addDistributedUri(localURI)) {
346347
val localPath = getQualifiedLocalPath(localURI, hadoopConf)
@@ -356,7 +357,7 @@ private[spark] class Client(
356357
(false, null)
357358
}
358359
} else {
359-
(true, path.trim())
360+
(true, trimmedPath)
360361
}
361362
}
362363

@@ -577,10 +578,10 @@ private[spark] class Client(
577578
LOCALIZED_PYTHON_DIR)
578579
}
579580
(pySparkArchives ++ pyArchives).foreach { path =>
580-
val uri = new URI(path)
581+
val uri = Utils.resolveURI(path)
581582
if (uri.getScheme != LOCAL_SCHEME) {
582583
pythonPath += buildPath(YarnSparkHadoopUtil.expandEnvironment(Environment.PWD),
583-
new Path(path).getName())
584+
new Path(uri).getName())
584585
} else {
585586
pythonPath += uri.getPath()
586587
}
@@ -1196,7 +1197,7 @@ object Client extends Logging {
11961197

11971198
private def getMainJarUri(mainJar: Option[String]): Option[URI] = {
11981199
mainJar.flatMap { path =>
1199-
val uri = new URI(path)
1200+
val uri = Utils.resolveURI(path)
12001201
if (uri.getScheme == LOCAL_SCHEME) Some(uri) else None
12011202
}.orElse(Some(new URI(APP_JAR)))
12021203
}

0 commit comments

Comments
 (0)