From 4ac9cacb5fc0ef3613fb7599cbf352f07ae4e9a9 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Sun, 27 Apr 2014 22:20:18 -0700 Subject: [PATCH 1/3] SPARK-1652: spark-submit for yarn prints warnings even though calling as expected --- .../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 ++ .../main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 ++++-- .../main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 24edc60684376..b0bade5cd5980 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -124,6 +124,8 @@ object SparkSubmit { } } else if (clusterManager == YARN) { childMainClass = "org.apache.spark.deploy.yarn.Client" + // Special flag to avoid deprecation warnings at the client + sysProps("SPARK_SUBMIT_YARN") = "true" childArgs += ("--jar", appArgs.primaryResource) childArgs += ("--class", appArgs.mainClass) } diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 00c7649e68e13..ee0435b04c481 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -169,8 +169,10 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa object Client { def main(argStrings: Array[String]) { - println("WARNING: This client is deprecated and will be removed in a future version of Spark.") - println("Use ./bin/spark-submit with \"--master yarn\"") + if (!sys.props.contains("SPARK_SUBMIT_YARN")) { + println("WARNING: This client is deprecated and will be removed in a " + + "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"") + } // Set an env variable indicating we are running in YARN mode. // Note that anything with SPARK prefix gets propagated to all (remote) processes diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 2e2fb5d4fa787..c1d9ae6817034 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -171,8 +171,10 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa object Client { def main(argStrings: Array[String]) { - println("WARNING: This client is deprecated and will be removed in a future version of Spark.") - println("Use ./bin/spark-submit with \"--master yarn\"") + if (!sys.props.contains("SPARK_SUBMIT_YARN")) { + println("WARNING: This client is deprecated and will be removed in a " + + "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"") + } // Set an env variable indicating we are running in YARN mode. // Note: anything env variable with SPARK_ prefix gets propagated to all (remote) processes - From b5be17360a6afca054884a61635612320920a8bc Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Mon, 28 Apr 2014 10:22:14 -0700 Subject: [PATCH 2/3] Review feedback --- core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 2 +- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index b0bade5cd5980..1b88f3343177d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -125,7 +125,7 @@ object SparkSubmit { } else if (clusterManager == YARN) { childMainClass = "org.apache.spark.deploy.yarn.Client" // Special flag to avoid deprecation warnings at the client - sysProps("SPARK_SUBMIT_YARN") = "true" + sysProps("SPARK_SUBMIT") = "true" childArgs += ("--jar", appArgs.primaryResource) childArgs += ("--class", appArgs.mainClass) } diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index ee0435b04c481..8226207de42b8 100644 --- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -169,7 +169,7 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa object Client { def main(argStrings: Array[String]) { - if (!sys.props.contains("SPARK_SUBMIT_YARN")) { + if (!sys.props.contains("SPARK_SUBMIT")) { println("WARNING: This client is deprecated and will be removed in a " + "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"") } diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index c1d9ae6817034..1b6bfb42a5c1c 100644 --- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -171,7 +171,7 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa object Client { def main(argStrings: Array[String]) { - if (!sys.props.contains("SPARK_SUBMIT_YARN")) { + if (!sys.props.contains("SPARK_SUBMIT")) { println("WARNING: This client is deprecated and will be removed in a " + "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"") } From 96027c71c8fb451af8f1fb3ab6a2c882f367099a Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Mon, 28 Apr 2014 11:54:49 -0700 Subject: [PATCH 3/3] Test fixes --- .../src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 5 +++-- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 1b88f3343177d..85a96efdbb17d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -124,12 +124,13 @@ object SparkSubmit { } } else if (clusterManager == YARN) { childMainClass = "org.apache.spark.deploy.yarn.Client" - // Special flag to avoid deprecation warnings at the client - sysProps("SPARK_SUBMIT") = "true" childArgs += ("--jar", appArgs.primaryResource) childArgs += ("--class", appArgs.mainClass) } + // Special flag to avoid deprecation warnings at the client + sysProps("SPARK_SUBMIT") = "true" + val options = List[OptionAssigner]( new OptionAssigner(appArgs.master, ALL_CLUSTER_MGRS, false, sysProp = "spark.master"), new OptionAssigner(appArgs.driverExtraClassPath, STANDALONE | YARN, true, diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 10a65c75cc621..b3541b4a40b79 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -122,7 +122,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { childArgsStr should include ("--num-executors 6") mainClass should be ("org.apache.spark.deploy.yarn.Client") classpath should have length (0) - sysProps should have size (0) + sysProps should have size (1) } test("handles YARN client mode") { @@ -146,6 +146,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { sysProps("spark.yarn.dist.files") should be ("file1.txt,file2.txt") sysProps("spark.yarn.dist.archives") should be ("archive1.txt,archive2.txt") sysProps("spark.executor.instances") should be ("6") + sysProps("SPARK_SUBMIT") should be ("true") } test("handles standalone cluster mode") { @@ -159,7 +160,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { childArgsStr should include ("launch spark://h:p thejar.jar org.SomeClass arg1 arg2") mainClass should be ("org.apache.spark.deploy.Client") classpath should have length (0) - sysProps should have size (1) // contains --jar entry + sysProps should have size (2) // contains --jar entry and SPARK_SUBMIT } test("handles standalone client mode") {