From e85a24d7fed6f79d60a79f7642d743f3b2b491a3 Mon Sep 17 00:00:00 2001 From: Phoenix_Dad Date: Mon, 22 May 2017 16:40:28 +0800 Subject: [PATCH] [SPARK- ][LAUNCHER] DRIVER_EXTRA_JAVA_OPTIONS is needed only when userDeployMode is 'client'. If userDeployMode is not 'client', the launcher shouldn't care about DRIVER_EXTRA_JAVA_OPTIONS, nor should it exit abnormally because DRIVER_EXTRA_JAVA_OPTIONS contains Xmx configuration. --- .../launcher/SparkSubmitCommandBuilder.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index 5f2da036ff9f7..f2596de4eb134 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -241,17 +241,16 @@ private List buildSparkSubmitCommand(Map env) } addOptionString(cmd, System.getenv("SPARK_SUBMIT_OPTS")); - // We don't want the client to specify Xmx. These have to be set by their corresponding - // memory flag --driver-memory or configuration entry spark.driver.memory - String driverExtraJavaOptions = config.get(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS); - if (!isEmpty(driverExtraJavaOptions) && driverExtraJavaOptions.contains("Xmx")) { - String msg = String.format("Not allowed to specify max heap(Xmx) memory settings through " + - "java options (was %s). Use the corresponding --driver-memory or " + - "spark.driver.memory configuration instead.", driverExtraJavaOptions); - throw new IllegalArgumentException(msg); - } - if (isClientMode) { + // We don't want the client to specify Xmx. These have to be set by their corresponding + // memory flag --driver-memory or configuration entry spark.driver.memory + String driverExtraJavaOptions = config.get(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS); + if (!isEmpty(driverExtraJavaOptions) && driverExtraJavaOptions.contains("Xmx")) { + String msg = String.format("Not allowed to specify max heap(Xmx) memory settings through " + + "java options (was %s). Use the corresponding --driver-memory or " + + "spark.driver.memory configuration instead.", driverExtraJavaOptions); + throw new IllegalArgumentException(msg); + } // Figuring out where the memory value come from is a little tricky due to precedence. // Precedence is observed in the following order: // - explicit configuration (setConf()), which also covers --driver-memory cli argument.