From 2c1b9e28986c32d68e17d99928ace4a620c59f60 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 00:23:12 -0700 Subject: [PATCH 01/11] Fix Executor's Command too long problem --- .../spark/deploy/worker/DriverRunner.scala | 5 ++ .../spark/deploy/worker/ExecutorRunner.scala | 5 ++ .../scala/org/apache/spark/util/Utils.scala | 68 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala index f4376dedea72..19b12fbfaca7 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/DriverRunner.scala @@ -181,6 +181,11 @@ private[deploy] class DriverRunner( Files.append(header, stderr, StandardCharsets.UTF_8) CommandUtils.redirectStream(process.getErrorStream, stderr) } + + if (Utils.isWindows) { + Utils.shortenClasspath(builder) + } + runCommandWithRetry(ProcessBuilderLike(builder), initialize, supervise) } diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala index 06066248ea5d..82983f8bc39d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala @@ -145,6 +145,11 @@ private[deploy] class ExecutorRunner( val builder = CommandUtils.buildProcessBuilder(appDesc.command, new SecurityManager(conf), memory, sparkHome.getAbsolutePath, substituteVariables) val command = builder.command() + + if (Utils.isWindows) { + Utils.shortenClasspath(builder) + } + val formattedCommand = command.asScala.mkString("\"", "\" \"", "\"") logInfo(s"Launch command: $formattedCommand") diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index ea49991493fd..25cbbbad2322 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -24,6 +24,7 @@ import java.nio.ByteBuffer import java.nio.channels.Channels import java.nio.charset.StandardCharsets import java.nio.file.Files +import java.util import java.util.{Locale, Properties, Random, UUID} import java.util.concurrent._ import javax.net.ssl.HttpsURLConnection @@ -1090,6 +1091,68 @@ private[spark] object Utils extends Logging { bytesToString(megabytes * 1024L * 1024L) } + + /** + * Create a jar file at the given path, containing a manifest with a classpath + * that references all specified entries. + */ + def createShortClassPath(tempDir: File, classPath: String) : String = { + if (isWindows) { + val env = new util.HashMap[String, String](System.getenv()) + val javaCp = FileUtil + .createJarWithClassPath(classPath, new Path(tempDir.getAbsolutePath), env) + .mkString(File.pathSeparator) + logInfo("Shorten the class path to: " + javaCp) + javaCp + } else { + classPath + } + } + + + def createShortClassPath(classPath: String) : String = { + val tempDir = createTempDir("classpaths") + createShortClassPath(tempDir, classPath) + } + + /** + * Create a jar file at the given path, containing a manifest with a classpath + * that references all specified entries. + */ + def shortenClasspath(builder: ProcessBuilder): Unit = { + if (builder.command.asScala.mkString("\"", "\" \"", "\"").length > 8190) { + logWarning("Cmd too long, try to shorten the classpath") + // look for the class path + // note that environment set in teh ProcessBuilder is process-local. So it + // won't pollute the environment + val command = builder.command() + val idxCp = command.indexOf("-cp") + if (idxCp > 0 && idxCp + 1 < command.size()) { + val classPath = command.get(idxCp + 1) + val shortPath = createShortClassPath(classPath) + command.set(idxCp + 1, shortPath) + } + } + } + + + /** + * Normalize the local path for windows. If the path is absolute (starts with drive label) + * it will append a leading slash. The back slashes will be replaced with slash + * @param path the path to be normalized + * @return the normalized path + */ + def normalizePath(path: String): String = { + if (isWindows) { + if (path.matches("""^[A-Za-z]:[/\\].*$""")) { + "/" + path.replace("\\", "/") + } else { + path.replace("\\", "/") + } + } else { + path + } + } /** * Execute a command and return the process running the command. */ @@ -1103,6 +1166,11 @@ private[spark] object Utils extends Logging { for ((key, value) <- extraEnvironment) { environment.put(key, value) } + + if (Utils.isWindows) { + Utils.shortenClasspath(builder) + } + val process = builder.start() if (redirectStderr) { val threadName = "redirect stderr for command " + command(0) From 79560bc2ccc4ce920982a72c7d1ec7568d0409f6 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 00:33:43 -0700 Subject: [PATCH 02/11] Fix Spark Launcher Suite --- .../spark/launcher/SparkLauncherSuite.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index 3e47bfc274cb..0da7760a0e9a 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -17,10 +17,13 @@ package org.apache.spark.launcher; +import java.io.File; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang.SystemUtils; +import org.apache.spark.util.Utils; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,16 +102,27 @@ public void testChildProcLauncher() throws Exception { String.format("%s=-Dfoo=ShouldBeOverriddenBelow", SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS)) .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Dfoo=bar -Dtest.appender=childproc") - .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) .addAppArgs("proc"); - final Process app = launcher.launch(); - - new OutputRedirector(app.getInputStream(), TF); - new OutputRedirector(app.getErrorStream(), TF); - assertEquals(0, app.waitFor()); - } + File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); + try { + if (SystemUtils.IS_OS_WINDOWS) { + launcher.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, + Utils.createShortClassPath(tempDir, System.getProperty("java.class.path"))); + } else { + launcher.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")); + } + final Process app = launcher.launch(); + + new OutputRedirector(app.getInputStream(), TF); + new OutputRedirector(app.getErrorStream(), TF); + assertEquals(0, app.waitFor()); + } finally { + if(tempDir.exists()) { + Utils.deleteRecursively(tempDir); + } + } } public static class SparkLauncherTestApp { From d092ef58bc889ec19bb5e4064cb751da9e58812c Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 00:48:39 -0700 Subject: [PATCH 03/11] Fix another too long --- .../spark/launcher/LauncherBackendSuite.scala | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala b/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala index 713560d3ddfa..ffca8c58f6dc 100644 --- a/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala +++ b/core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala @@ -26,6 +26,7 @@ import org.scalatest.Matchers import org.scalatest.concurrent.Eventually._ import org.apache.spark._ +import org.apache.spark.util.Utils class LauncherBackendSuite extends SparkFunSuite with Matchers { @@ -33,6 +34,8 @@ class LauncherBackendSuite extends SparkFunSuite with Matchers { "local" -> "local", "standalone/client" -> "local-cluster[1,1,1024]") + val tempDir = Utils.createTempDir() + tests.foreach { case (name, master) => test(s"$name: launcher handle") { testWithMaster(master) @@ -42,16 +45,22 @@ class LauncherBackendSuite extends SparkFunSuite with Matchers { private def testWithMaster(master: String): Unit = { val env = new java.util.HashMap[String, String]() env.put("SPARK_PRINT_LAUNCH_COMMAND", "1") - val handle = new SparkLauncher(env) + + val launcher = new SparkLauncher(env) .setSparkHome(sys.props("spark.test.home")) - .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) .setConf("spark.ui.enabled", "false") .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, s"-Dtest.appender=console") .setMaster(master) .setAppResource("spark-internal") .setMainClass(TestApp.getClass.getName().stripSuffix("$")) - .startApplication() + if (Utils.isWindows) { + launcher.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, + Utils.createShortClassPath(tempDir, System.getProperty("java.class.path"))) + } else { + launcher.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path")) + } + val handle = launcher.startApplication() try { eventually(timeout(30 seconds), interval(100 millis)) { handle.getAppId() should not be (null) From 6f8671ee6264ddcb0b45d395f4a5f911ec1e52e2 Mon Sep 17 00:00:00 2001 From: Tao LI Date: Mon, 25 Apr 2016 01:18:11 -0700 Subject: [PATCH 04/11] Fix Abstract cmd builder --- .../launcher/AbstractCommandBuilder.java | 157 +++++++++++++++++- .../SparkSubmitCommandBuilderSuite.java | 54 +++++- 2 files changed, 204 insertions(+), 7 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index c7488082ca89..bbf90b08f028 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -18,8 +18,10 @@ package org.apache.spark.launcher; import java.io.BufferedReader; +import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.InputStreamReader; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -28,8 +30,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; import java.util.Properties; import java.util.regex.Pattern; +import java.util.regex.Matcher; import static org.apache.spark.launcher.CommandBuilderUtils.*; @@ -53,6 +59,8 @@ abstract class AbstractCommandBuilder { final Map childEnv; final Map conf; + private static File classPathShortDir = new File(System.getProperty("java.io.tmpdir"), "spark-classpath"); + // The merged configuration for the application. Cached to avoid having to read / parse // properties files multiple times. private Map effectiveConfig; @@ -115,10 +123,157 @@ List buildJavaCommand(String extraClassPath) throws IOException { } cmd.add("-cp"); - cmd.add(join(File.pathSeparator, buildClassPath(extraClassPath))); + List classPathEntries = buildClassPath(extraClassPath); + String classPath = null; + if(isWindows()) { + String[] jarCp = createJarWithClassPath( + classPathEntries.toArray(new String[classPathEntries.size()]), + classPathShortDir, + new HashMap<>(System.getenv())); + classPath = jarCp[0] + File.pathSeparator + jarCp[1]; + } else { + classPath = join(File.pathSeparator, classPathEntries); + } + cmd.add(classPath); return cmd; } + /** + * Create a jar file at the given path, containing a manifest with a classpath + * that references all specified entries. + * + * Some platforms may have an upper limit on command line length. For example, + * the maximum command line length on Windows is 8191 characters, but the + * length of the classpath may exceed this. To work around this limitation, + * use this method to create a small intermediate jar with a manifest that + * contains the full classpath. It returns the absolute path to the new jar, + * which the caller may set as the classpath for a new process. + * + * Environment variable evaluation is not supported within a jar manifest, so + * this method expands environment variables before inserting classpath entries + * to the manifest. The method parses environment variables according to + * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, + * environment variables are case-insensitive. For example, %VAR% and %var% + * evaluate to the same value. + * + * Specifying the classpath in a jar manifest does not support wildcards, so + * this method expands wildcards internally. Any classpath entry that ends + * with * is translated to all files at that path with extension .jar or .JAR. + * + * This method is adapted from the Hadoop-common-project FSUtils#createJarWithClassPath + * Reimplement the mehtod here to avoid heavy dependencies + * + * @param classPathEntries String input classpath to bundle into the jar manifest + * @param workingDir Path to working directory to save jar + * @param callerEnv Map caller's environment variables to use + * for expansion + * @return String[] with absolute path to new jar in position 0 and + * unexpanded wild card entry path in position 1 + * @throws IOException if there is an I/O error while writing the jar file + */ + public static String[] createJarWithClassPath(String[] classPathEntries, File workingDir, + Map callerEnv) throws IOException { + // Replace environment variables, case-insensitive on Windows + Map env = new HashMap<>(); + if (isWindows()) { + for(Map.Entry entry : callerEnv.entrySet()) { + env.put(entry.getKey().toLowerCase(), entry.getValue()); + } + } else { + env = callerEnv; + } + + // expand the environment variables + Pattern envVarPattern = isWindows() ? Pattern.compile("%(.*?)%") : + Pattern.compile("\\$([A-Za-z_]{1}[A-Za-z0-9_]*)"); + for (int i = 0; i < classPathEntries.length; ++i) { + String template = classPathEntries[i]; + StringBuffer sb = new StringBuffer(); + Matcher matcher = envVarPattern.matcher(template); + while (matcher.find()) { + String replacement = env.get(matcher.group(1)); + if (replacement == null) { + replacement = ""; + } + matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(sb); + classPathEntries[i] = sb.toString(); + } + + if (!workingDir.exists()) { + workingDir.mkdirs(); + } + + StringBuilder unexpandedWildcardClasspath = new StringBuilder(); + // Append all entries + List classPathEntryList = new ArrayList( + classPathEntries.length); + for (String classPathEntry: classPathEntries) { + if (classPathEntry.length() == 0) { + continue; + } + if (classPathEntry.endsWith("*")) { + boolean foundWildCardJar = false; + // Append all jars that match the wildcard + File[] files = new File(classPathEntry.substring(0, classPathEntry.length() - 2)).listFiles(); + for(File f: files) { + if (f.getName().toLowerCase().endsWith("jar")) { + foundWildCardJar = true; + classPathEntryList.add(f.getAbsoluteFile().toURI().toURL().toExternalForm()); + } + } + + if (!foundWildCardJar) { + unexpandedWildcardClasspath.append(File.pathSeparator); + unexpandedWildcardClasspath.append(classPathEntry); + } + } else { + // Append just this entry + File fileCpEntry = new File(classPathEntry); + String classPathEntryUrl = fileCpEntry.toURI().toURL() + .toExternalForm(); + + // File.toURI only appends trailing '/' if it can determine that it is a + // directory that already exists. (See JavaDocs.) If this entry had a + // trailing '/' specified by the caller, then guarantee that the + // classpath entry in the manifest has a trailing '/', and thus refers to + // a directory instead of a file. This can happen if the caller is + // creating a classpath jar referencing a directory that hasn't been + // created yet, but will definitely be created before running. + if (classPathEntry.endsWith("/") && + !classPathEntryUrl.endsWith("/")) { + classPathEntryUrl = classPathEntryUrl + "/"; + } + classPathEntryList.add(classPathEntryUrl); + } + } + String jarClassPath = join(" ", classPathEntryList); + + // Create the manifest + Manifest jarManifest = new Manifest(); + jarManifest.getMainAttributes().putValue( + Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); + jarManifest.getMainAttributes().putValue( + Attributes.Name.CLASS_PATH.toString(), jarClassPath); + + // Write the manifest to output JAR file + File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); + FileOutputStream fos = null; + BufferedOutputStream bos = null; + JarOutputStream jos = null; + try { + fos = new FileOutputStream(classPathJar); + bos = new BufferedOutputStream(fos); + jos = new JarOutputStream(bos, jarManifest); + } finally { + jos.close(); + } + String[] jarCp = {classPathJar.getCanonicalPath(), + unexpandedWildcardClasspath.toString()}; + return jarCp; + } + void addOptionString(List cmd, String options) { if (!isEmpty(options)) { for (String opt : parseOptionString(options)) { diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index c7e8b2e03a9f..d89c184726ef 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -23,8 +23,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; import java.util.regex.Pattern; +import com.sun.javafx.util.Utils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -66,7 +70,7 @@ public void testCliParser() throws Exception { parser.DRIVER_MEMORY, "42g", parser.DRIVER_CLASS_PATH, - "/driverCp", + Utils.isWindows() ? "D:/driverCp" : "/driverCp", parser.DRIVER_JAVA_OPTIONS, "extraJavaOpt", parser.CONF, @@ -78,7 +82,25 @@ public void testCliParser() throws Exception { assertTrue(findInStringList(env.get(CommandBuilderUtils.getLibPathEnvName()), File.pathSeparator, "/driverLibPath")); - assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp")); + if (!Utils.isWindows()) + { + assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp")); + } else { + String[] cp = findArgValue(cmd, "-cp").split(File.pathSeparator); + JarFile jarFile = null; + try { + jarFile = new JarFile(cp[0]); + Manifest jarManifest = jarFile.getManifest(); + Attributes attrs = jarManifest.getMainAttributes(); + String classPath = attrs.getValue(Attributes.Name.CLASS_PATH); + assertTrue(contains("file:/D:/driverCp", classPath.split(" "))); + } finally { + if (jarFile != null) { + jarFile.close(); + } + + } + } assertTrue("Driver -Xmx should be configured.", cmd.contains("-Xmx42g")); assertTrue("Command should contain user-defined conf.", Collections.indexOfSubList(cmd, Arrays.asList(parser.CONF, "spark.randomOption=foo")) > 0); @@ -187,7 +209,7 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th if (!useDefaultPropertyFile) { launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath()); launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g"); - launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver"); + launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, Utils.isWindows() ? "D:/driver": "/driver"); launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m"); launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native"); } else { @@ -220,10 +242,30 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th } String[] cp = findArgValue(cmd, "-cp").split(Pattern.quote(File.pathSeparator)); - if (isDriver) { - assertTrue("Driver classpath should contain provided entry.", contains("/driver", cp)); + if (!Utils.isWindows()) { + if (isDriver) { + assertTrue("Driver classpath should contain provided entry.", contains("/driver", cp)); + } else { + assertFalse("Driver classpath should not be in command.", contains("/driver", cp)); + } } else { - assertFalse("Driver classpath should not be in command.", contains("/driver", cp)); + JarFile jarFile = null; + try { + jarFile = new JarFile(cp[0]); + Manifest jarManifest = jarFile.getManifest(); + Attributes attrs = jarManifest.getMainAttributes(); + String classPath = attrs.getValue(Attributes.Name.CLASS_PATH); + if (isDriver) { + assertTrue(contains("file:/D:/driver", classPath.split(" "))); + } else { + assertFalse(contains("file:/D:/driver", classPath.split(" "))); + } + } finally { + if (jarFile != null) { + jarFile.close(); + } + + } } String libPath = env.get(CommandBuilderUtils.getLibPathEnvName()); From efc0df0a93b668829a67afcbf0f2bd2ed3e60450 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 01:47:00 -0700 Subject: [PATCH 05/11] Remove normalize path --- .../scala/org/apache/spark/util/Utils.scala | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 25cbbbad2322..3e0cb1957672 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1136,23 +1136,6 @@ private[spark] object Utils extends Logging { } - /** - * Normalize the local path for windows. If the path is absolute (starts with drive label) - * it will append a leading slash. The back slashes will be replaced with slash - * @param path the path to be normalized - * @return the normalized path - */ - def normalizePath(path: String): String = { - if (isWindows) { - if (path.matches("""^[A-Za-z]:[/\\].*$""")) { - "/" + path.replace("\\", "/") - } else { - path.replace("\\", "/") - } - } else { - path - } - } /** * Execute a command and return the process running the command. */ From df0f5dd67196cb8aa4deba49eca254a3dafe8e1f Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 02:11:07 -0700 Subject: [PATCH 06/11] Minor format fix --- .../java/org/apache/spark/launcher/SparkLauncherSuite.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index 0da7760a0e9a..131e368a245e 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -105,7 +105,8 @@ public void testChildProcLauncher() throws Exception { .addSparkArg(opts.CLASS, "ShouldBeOverriddenBelow") .setMainClass(SparkLauncherTestApp.class.getName()) .addAppArgs("proc"); - File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); + + File tempDir = Utils.createTempDir(System.getProperty("java.io.tmpdir"), "spark"); try { if (SystemUtils.IS_OS_WINDOWS) { launcher.setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, @@ -122,7 +123,8 @@ public void testChildProcLauncher() throws Exception { if(tempDir.exists()) { Utils.deleteRecursively(tempDir); } - } } + } + } public static class SparkLauncherTestApp { From e26d9495a9d8f8f237c2328f3028185446b5370f Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 03:17:00 -0700 Subject: [PATCH 07/11] fix spark-submit.cmd --- bin/spark-submit.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/spark-submit.cmd b/bin/spark-submit.cmd index f301606933a9..e2c69616d0a8 100644 --- a/bin/spark-submit.cmd +++ b/bin/spark-submit.cmd @@ -20,4 +20,4 @@ rem rem This is the entry point for running Spark submit. To avoid polluting the rem environment, it just launches a new cmd to do the real work. -cmd /V /E /C "%~dp0spark-submit2.cmd" %* +cmd /V /E /C ""%~dp0spark-submit2.cmd" %*" From 4a7fb864e337db5b395d092a94085bbf92853c8d Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 03:47:12 -0700 Subject: [PATCH 08/11] Fix shorten class path --- core/src/main/scala/org/apache/spark/util/Utils.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 3e0cb1957672..86fc6c14f272 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -1099,17 +1099,16 @@ private[spark] object Utils extends Logging { def createShortClassPath(tempDir: File, classPath: String) : String = { if (isWindows) { val env = new util.HashMap[String, String](System.getenv()) - val javaCp = FileUtil + val javaCps = FileUtil .createJarWithClassPath(classPath, new Path(tempDir.getAbsolutePath), env) - .mkString(File.pathSeparator) - logInfo("Shorten the class path to: " + javaCp) - javaCp + val javaCpStr = javaCps(0) + javaCps(1) + logInfo("Shorten the class path to: " + javaCpStr) + javaCpStr } else { classPath } } - def createShortClassPath(classPath: String) : String = { val tempDir = createTempDir("classpaths") createShortClassPath(tempDir, classPath) From d40d61a04628c5729c3ebc10c46ec525bd77a0eb Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 20:38:26 -0700 Subject: [PATCH 09/11] Handle null files on create jar --- .../apache/spark/launcher/AbstractCommandBuilder.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index bbf90b08f028..ea46c2da9ba5 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -217,10 +217,12 @@ public static String[] createJarWithClassPath(String[] classPathEntries, File wo boolean foundWildCardJar = false; // Append all jars that match the wildcard File[] files = new File(classPathEntry.substring(0, classPathEntry.length() - 2)).listFiles(); - for(File f: files) { - if (f.getName().toLowerCase().endsWith("jar")) { - foundWildCardJar = true; - classPathEntryList.add(f.getAbsoluteFile().toURI().toURL().toExternalForm()); + if (files != null) { + for (File f : files) { + if (f.getName().toLowerCase().endsWith("jar")) { + foundWildCardJar = true; + classPathEntryList.add(f.getAbsoluteFile().toURI().toURL().toExternalForm()); + } } } From 41d0003d4a2fda5b3220ec5799f27b489d400dfa Mon Sep 17 00:00:00 2001 From: Tao LI Date: Tue, 3 May 2016 14:31:18 +0800 Subject: [PATCH 10/11] Scala style fixes --- core/src/main/scala/org/apache/spark/util/Utils.scala | 6 +++--- .../java/org/apache/spark/launcher/SparkLauncherSuite.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 86fc6c14f272..4a4f7f3d6cac 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -110,7 +110,7 @@ private[spark] object Utils extends Logging { } /** Deserialize a Long value (used for [[org.apache.spark.api.python.PythonPartitioner]]) */ - def deserializeLongValue(bytes: Array[Byte]) : Long = { + def deserializeLongValue(bytes: Array[Byte]): Long = { // Note: we assume that we are given a Long value encoded in network (big-endian) byte order var result = bytes(7) & 0xFFL result = result + ((bytes(6) & 0xFFL) << 8) @@ -1096,7 +1096,7 @@ private[spark] object Utils extends Logging { * Create a jar file at the given path, containing a manifest with a classpath * that references all specified entries. */ - def createShortClassPath(tempDir: File, classPath: String) : String = { + def createShortClassPath(tempDir: File, classPath: String): String = { if (isWindows) { val env = new util.HashMap[String, String](System.getenv()) val javaCps = FileUtil @@ -1109,7 +1109,7 @@ private[spark] object Utils extends Logging { } } - def createShortClassPath(classPath: String) : String = { + def createShortClassPath(classPath: String): String = { val tempDir = createTempDir("classpaths") createShortClassPath(tempDir, classPath) } diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index 131e368a245e..02543407b6f0 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -23,13 +23,14 @@ import java.util.Map; import org.apache.commons.lang.SystemUtils; -import org.apache.spark.util.Utils; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.bridge.SLF4JBridgeHandler; import static org.junit.Assert.*; +import org.apache.spark.util.Utils; + /** * These tests require the Spark assembly to be built before they can be run. */ From a0b353190e4eca86c5985087154b3f8a0c60fd5a Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Tue, 3 May 2016 02:50:29 -0700 Subject: [PATCH 11/11] Remove the dependency to javafx --- .../spark/launcher/SparkSubmitCommandBuilderSuite.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index d89c184726ef..3a641342aded 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -28,7 +28,6 @@ import java.util.jar.Manifest; import java.util.regex.Pattern; -import com.sun.javafx.util.Utils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -70,7 +69,7 @@ public void testCliParser() throws Exception { parser.DRIVER_MEMORY, "42g", parser.DRIVER_CLASS_PATH, - Utils.isWindows() ? "D:/driverCp" : "/driverCp", + CommandBuilderUtils.isWindows() ? "D:/driverCp" : "/driverCp", parser.DRIVER_JAVA_OPTIONS, "extraJavaOpt", parser.CONF, @@ -82,7 +81,7 @@ public void testCliParser() throws Exception { assertTrue(findInStringList(env.get(CommandBuilderUtils.getLibPathEnvName()), File.pathSeparator, "/driverLibPath")); - if (!Utils.isWindows()) + if (!CommandBuilderUtils.isWindows()) { assertTrue(findInStringList(findArgValue(cmd, "-cp"), File.pathSeparator, "/driverCp")); } else { @@ -209,7 +208,7 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th if (!useDefaultPropertyFile) { launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath()); launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g"); - launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, Utils.isWindows() ? "D:/driver": "/driver"); + launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, CommandBuilderUtils.isWindows() ? "D:/driver": "/driver"); launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m"); launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native"); } else { @@ -242,7 +241,7 @@ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) th } String[] cp = findArgValue(cmd, "-cp").split(Pattern.quote(File.pathSeparator)); - if (!Utils.isWindows()) { + if (!CommandBuilderUtils.isWindows()) { if (isDriver) { assertTrue("Driver classpath should contain provided entry.", contains("/driver", cp)); } else {