From ff6c162c4f7a7294e8b4465bf6ed083079603900 Mon Sep 17 00:00:00 2001 From: Reynold Xin Date: Mon, 26 May 2014 20:15:43 -0700 Subject: [PATCH 1/2] SPARK-1933: Throw a more meaningful exception when a directory is passed to addJar/addFile. --- core/src/main/scala/org/apache/spark/HttpFileServer.scala | 6 ++++++ core/src/main/scala/org/apache/spark/SparkContext.scala | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/HttpFileServer.scala b/core/src/main/scala/org/apache/spark/HttpFileServer.scala index a6e300d345786..e77c341850d36 100644 --- a/core/src/main/scala/org/apache/spark/HttpFileServer.scala +++ b/core/src/main/scala/org/apache/spark/HttpFileServer.scala @@ -59,6 +59,12 @@ private[spark] class HttpFileServer(securityManager: SecurityManager) extends Lo } def addFileToDir(file: File, dir: File) : String = { + // Check whether the file is a directory. If it is, throw a more meaningful exception. + // If we don't catch this, Guava throws an confusing error message when [file] is a directory: + // java.io.FileNotFoundException: [file] (No such file or directory) + if (file.isDirectory) { + throw new IllegalArgumentException(s"$file cannot be a directory.") + } Files.copy(file, new File(dir, file.getName)) dir + "/" + file.getName } diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 49737fa4be56b..03ceff8bf1fb0 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -794,7 +794,7 @@ class SparkContext(config: SparkConf) extends Logging { addedFiles(key) = System.currentTimeMillis // Fetch the file locally in case a job is executed using DAGScheduler.runLocally(). - Utils.fetchFile(path, new File(SparkFiles.getRootDirectory), conf, env.securityManager) + Utils.fetchFile(path, new File(SparkFiles.getRootDirectory()), conf, env.securityManager) logInfo("Added file " + path + " at " + key + " with timestamp " + addedFiles(key)) postEnvironmentUpdate() @@ -932,13 +932,12 @@ class SparkContext(config: SparkConf) extends Logging { try { env.httpFileServer.addJar(new File(fileName)) } catch { - case e: Exception => { + case e: Exception => // For now just log an error but allow to go through so spark examples work. // The spark examples don't really need the jar distributed since its also // the app jar. logError("Error adding jar (" + e + "), was the --addJars option used?") null - } } } else { env.httpFileServer.addJar(new File(uri.getPath)) From 8c402a38ad8c01b08a33a99c7af834605f763d02 Mon Sep 17 00:00:00 2001 From: Reynold Xin Date: Mon, 26 May 2014 20:19:01 -0700 Subject: [PATCH 2/2] Updated comment. --- core/src/main/scala/org/apache/spark/HttpFileServer.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/HttpFileServer.scala b/core/src/main/scala/org/apache/spark/HttpFileServer.scala index e77c341850d36..0e3750fdde415 100644 --- a/core/src/main/scala/org/apache/spark/HttpFileServer.scala +++ b/core/src/main/scala/org/apache/spark/HttpFileServer.scala @@ -60,8 +60,9 @@ private[spark] class HttpFileServer(securityManager: SecurityManager) extends Lo def addFileToDir(file: File, dir: File) : String = { // Check whether the file is a directory. If it is, throw a more meaningful exception. - // If we don't catch this, Guava throws an confusing error message when [file] is a directory: + // If we don't catch this, Guava throws a very confusing error message: // java.io.FileNotFoundException: [file] (No such file or directory) + // even though the directory ([file]) exists. if (file.isDirectory) { throw new IllegalArgumentException(s"$file cannot be a directory.") }