From de790fd251ba3727bba23ceb1ca07559d25b7e87 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Tue, 29 May 2018 11:03:40 -0700 Subject: [PATCH 1/8] Upgrade scala to 2.11.12 and 2.12.6 --- dev/deps/spark-deps-hadoop-2.6 | 6 +++--- dev/deps/spark-deps-hadoop-2.7 | 6 +++--- dev/deps/spark-deps-hadoop-3.1 | 6 +++--- pom.xml | 4 ++-- .../org/apache/spark/repl/SparkILoop.scala | 21 ++++--------------- .../spark/repl/SparkILoopInterpreter.scala | 18 ++++++++++++++-- 6 files changed, 31 insertions(+), 30 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 723180a14febb..b64b6c2cbd454 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -172,10 +172,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index ea08a001a1c9b..fbcb824c61bd6 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -173,10 +173,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index da874026d7d10..25b312623e5ea 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -192,10 +192,10 @@ protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar re2j-1.1.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/pom.xml b/pom.xml index 883c096ae1ae9..15947abe770ea 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 3.4.1 3.2.2 - 2.11.8 + 2.11.12 2.11 1.9.13 2.6.7 @@ -2748,7 +2748,7 @@ scala-2.12 - 2.12.4 + 2.12.6 2.12 diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index e69441a475e9a..226ef7067732b 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -36,7 +36,7 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) def this() = this(None, new JPrintWriter(Console.out, true)) override def createInterpreter(): Unit = { - intp = new SparkILoopInterpreter(settings, out) + intp = new SparkILoopInterpreter(settings, out, initializeSpark) } val initializationCommands: Seq[String] = Seq( @@ -73,12 +73,9 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) "import org.apache.spark.sql.functions._" ) - def initializeSpark() { - intp.beQuietDuring { - savingReplayStack { // remove the commands from session history. - initializationCommands.foreach(processLine) - } - } + def initializeSpark(): Unit = savingReplayStack { + // `savingReplayStack` removes the commands from session history. + initializationCommands.foreach(intp quietRun _) } /** Print a welcome message */ @@ -101,16 +98,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) /** Available commands */ override def commands: List[LoopCommand] = standardCommands - /** - * We override `loadFiles` because we need to initialize Spark *before* the REPL - * sees any files, so that the Spark context is visible in those files. This is a bit of a - * hack, but there isn't another hook available to us at this point. - */ - override def loadFiles(settings: Settings): Unit = { - initializeSpark() - super.loadFiles(settings) - } - override def resetCommand(line: String): Unit = { super.resetCommand(line) initializeSpark() diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index e736607a9a6b9..4e63816402a10 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -21,8 +21,22 @@ import scala.collection.mutable import scala.tools.nsc.Settings import scala.tools.nsc.interpreter._ -class SparkILoopInterpreter(settings: Settings, out: JPrintWriter) extends IMain(settings, out) { - self => +class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpark: () => Unit) + extends IMain(settings, out) { self => + + /** + * We override `initializeSynchronous` to initialize Spark *after* `intp` is properly initialized + * and *before* the REPL sees any files in the private `loadInitFiles` functions, so that + * the Spark context is visible in those files. + * + * This is a bit of a hack, but there isn't another hook available to us at this point. + * + * See the discussion in Scala community https://github.com/scala/bug/issues/10913 for detail. + */ + override def initializeSynchronous(): Unit = { + super.initializeSynchronous() + initializeSpark() + } override lazy val memberHandlers = new { val intp: self.type = self From f91d75ae6b77bef6bf7eb8db98a345e6eb822393 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Thu, 7 Jun 2018 17:48:21 -0700 Subject: [PATCH 2/8] Update LICENSE --- LICENSE | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index 820f14dbdeed0..1e10efeb5dab4 100644 --- a/LICENSE +++ b/LICENSE @@ -249,11 +249,11 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (Interpreter classes (all .scala files in repl/src/main/scala except for Main.Scala, SparkHelper.scala and ExecutorClassLoader.scala), and for SerializableMapWrapper in JavaUtils.scala) - (BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Library (org.scala-lang:scala-library:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scalap (org.scala-lang:scalap:2.11.8 - http://www.scala-lang.org/) + (BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Library (org.scala-lang:scala-library:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scalap (org.scala-lang:scalap:2.11.12 - http://www.scala-lang.org/) (BSD-style) scalacheck (org.scalacheck:scalacheck_2.11:1.10.0 - http://www.scalacheck.org) (BSD-style) spire (org.spire-math:spire_2.11:0.7.1 - http://spire-math.org) (BSD-style) spire-macros (org.spire-math:spire-macros_2.11:0.7.1 - http://spire-math.org) From c1ffd0b7bb0ae60a1d7740c9a3b00ee804e1f156 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Jun 2018 00:57:21 -0700 Subject: [PATCH 3/8] Update JLine --- LICENSE | 2 +- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- dev/deps/spark-deps-hadoop-3.1 | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index 1e10efeb5dab4..d3d920e6f2a06 100644 --- a/LICENSE +++ b/LICENSE @@ -242,7 +242,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (BSD licence) ANTLR ST4 4.0.4 (org.antlr:ST4:4.0.4 - http://www.stringtemplate.org) (BSD licence) ANTLR StringTemplate (org.antlr:stringtemplate:3.2.1 - http://www.stringtemplate.org) (BSD License) Javolution (javolution:javolution:5.5.1 - http://javolution.org) - (BSD) JLine (jline:jline:0.9.94 - http://jline.sourceforge.net) + (BSD) JLine (jline:jline:2.14.3 - https://github.com/jline/jline2) (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.3 - http://paranamer.codehaus.org/paranamer) (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.6 - http://paranamer.codehaus.org/paranamer) (BSD 3 Clause) Scala (http://www.scala-lang.org/download/#License) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index b64b6c2cbd454..4dfffa99d3097 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-6.1.26.jar jetty-util-6.1.26.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index fbcb824c61bd6..e32e79d63dff6 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-6.1.26.jar jetty-util-6.1.26.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 25b312623e5ea..3677c30132502 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-webapp-9.3.20.v20170531.jar jetty-xml-9.3.20.v20170531.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar diff --git a/pom.xml b/pom.xml index 15947abe770ea..2dfe2d6c08223 100644 --- a/pom.xml +++ b/pom.xml @@ -746,7 +746,7 @@ jline jline - 2.12.1 + 2.14.3 org.scalatest From 631ef48d63955ca231d2d559b0fe93d018e0cd8a Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Jun 2018 01:14:21 -0700 Subject: [PATCH 4/8] update scala-parser-combinators --- dev/deps/spark-deps-hadoop-2.6 | 2 +- dev/deps/spark-deps-hadoop-2.7 | 2 +- dev/deps/spark-deps-hadoop-3.1 | 2 +- pom.xml | 2 +- .../scala/org/apache/spark/repl/SparkILoopInterpreter.scala | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 4dfffa99d3097..96e9c27210d05 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -174,7 +174,7 @@ py4j-0.10.7.jar pyrolite-4.13.jar scala-compiler-2.11.12.jar scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.0.4.jar +scala-parser-combinators_2.11-1.1.0.jar scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index e32e79d63dff6..4a6ee027ec355 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -175,7 +175,7 @@ py4j-0.10.7.jar pyrolite-4.13.jar scala-compiler-2.11.12.jar scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.0.4.jar +scala-parser-combinators_2.11-1.1.0.jar scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 3677c30132502..e0b560c8ec71f 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -194,7 +194,7 @@ pyrolite-4.13.jar re2j-1.1.jar scala-compiler-2.11.12.jar scala-library-2.11.12.jar -scala-parser-combinators_2.11-1.0.4.jar +scala-parser-combinators_2.11-1.1.0.jar scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar diff --git a/pom.xml b/pom.xml index 2dfe2d6c08223..4ccdfb34d3be2 100644 --- a/pom.xml +++ b/pom.xml @@ -740,7 +740,7 @@ org.scala-lang.modules scala-parser-combinators_${scala.binary.version} - 1.0.4 + 1.1.0 diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index 4e63816402a10..0e23b03875f5f 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -22,7 +22,7 @@ import scala.tools.nsc.Settings import scala.tools.nsc.interpreter._ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpark: () => Unit) - extends IMain(settings, out) { self => + extends IMain(settings, out) { self => /** * We override `initializeSynchronous` to initialize Spark *after* `intp` is properly initialized From 96e87c2111a042fff3d5174326956775e41fd689 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Jun 2018 11:36:25 -0700 Subject: [PATCH 5/8] Error handling --- .../org/apache/spark/repl/SparkILoop.scala | 21 ++++++++++++++++--- .../spark/repl/SparkILoopInterpreter.scala | 8 ++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 226ef7067732b..0487ce414d8b8 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -73,13 +73,28 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) "import org.apache.spark.sql.functions._" ) - def initializeSpark(): Unit = savingReplayStack { - // `savingReplayStack` removes the commands from session history. - initializationCommands.foreach(intp quietRun _) + def initializeSpark(): Unit = { + if (!intp.reporter.hasErrors) { + // `savingReplayStack` removes the commands from session history. + savingReplayStack { + initializationCommands.foreach(intp quietRun _) + } + } else { + throw new RuntimeException(s"Scala $versionString interpreter encountered " + + "errors during initialization") + } } /** Print a welcome message */ override def printWelcome() { + // Before Scala 2.11.9, `printWelcome()` will be the last thing to be called, + // so Scala REPL and Spark will be initialized before `printWelcome()`. + // After Scala 2.11.9, `printWelcome()` will be the first thing to be called, + // as a result, Scala REPL and Spark will be initialized in `printWelcome()`. + if (!isInitializeComplete) { + intp.initializeSynchronous() + } + import org.apache.spark.SPARK_VERSION echo("""Welcome to ____ __ diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index 0e23b03875f5f..911445ca0c10f 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -22,7 +22,7 @@ import scala.tools.nsc.Settings import scala.tools.nsc.interpreter._ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpark: () => Unit) - extends IMain(settings, out) { self => + extends IMain(settings, out) { self => /** * We override `initializeSynchronous` to initialize Spark *after* `intp` is properly initialized @@ -34,8 +34,10 @@ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpa * See the discussion in Scala community https://github.com/scala/bug/issues/10913 for detail. */ override def initializeSynchronous(): Unit = { - super.initializeSynchronous() - initializeSpark() + if (!isInitializeComplete) { + super.initializeSynchronous() + initializeSpark() + } } override lazy val memberHandlers = new { From 4c852fa6086bc17871e3fa742f77620e51f809f8 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Jun 2018 14:01:31 -0700 Subject: [PATCH 6/8] Fix compilation --- .../scala/org/apache/spark/repl/SparkILoop.scala | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 0487ce414d8b8..3f7ed606ae552 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -87,11 +87,15 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) /** Print a welcome message */ override def printWelcome() { - // Before Scala 2.11.9, `printWelcome()` will be the last thing to be called, - // so Scala REPL and Spark will be initialized before `printWelcome()`. - // After Scala 2.11.9, `printWelcome()` will be the first thing to be called, - // as a result, Scala REPL and Spark will be initialized in `printWelcome()`. - if (!isInitializeComplete) { + // Before Scala 2.11.9, `printWelcome()` will be called after Scala REPL and Spark + // are initialized, so we will not call `initializeSynchronous()` in `printWelcome()`. + // + // However, after Scala 2.11.9, `printWelcome()` will be the first thing to be called, + // so we initialize the Scala REPL and Spark here instead. + if (intp == null) { + createInterpreter() + } + if (!intp.isInitializeComplete) { intp.initializeSynchronous() } From 096e6a9dd9af96108cee70b5eda097f9fa7afe01 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Jun 2018 17:17:17 -0700 Subject: [PATCH 7/8] temp --- .../src/main/scala/org/apache/spark/repl/SparkILoop.scala | 4 ++++ .../scala/org/apache/spark/repl/SparkILoopInterpreter.scala | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 3f7ed606ae552..096e88e80871b 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -95,10 +95,14 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) if (intp == null) { createInterpreter() } + echo(s"DB printWelcome 1 ${intp.isInitializeComplete}" ) + if (!intp.isInitializeComplete) { intp.initializeSynchronous() } + echo(s"DB printWelcome 2 ${intp.isInitializeComplete}" ) + import org.apache.spark.SPARK_VERSION echo("""Welcome to ____ __ diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index 911445ca0c10f..6f696ed7d8625 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -34,6 +34,10 @@ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpa * See the discussion in Scala community https://github.com/scala/bug/issues/10913 for detail. */ override def initializeSynchronous(): Unit = { + // scalastyle:off println + println(s"DB initializeSynchronous ${isInitializeComplete}" ) + // scalastyle:on println + if (!isInitializeComplete) { super.initializeSynchronous() initializeSpark() From 82ca5f6fdabc3027f2706af4b6883d04ac352555 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Sun, 10 Jun 2018 23:28:40 -0700 Subject: [PATCH 8/8] Revert hack --- .../scala/org/apache/spark/repl/SparkILoop.scala | 16 ---------------- .../spark/repl/SparkILoopInterpreter.scala | 10 ++-------- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 096e88e80871b..a44051b351e19 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -87,22 +87,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) /** Print a welcome message */ override def printWelcome() { - // Before Scala 2.11.9, `printWelcome()` will be called after Scala REPL and Spark - // are initialized, so we will not call `initializeSynchronous()` in `printWelcome()`. - // - // However, after Scala 2.11.9, `printWelcome()` will be the first thing to be called, - // so we initialize the Scala REPL and Spark here instead. - if (intp == null) { - createInterpreter() - } - echo(s"DB printWelcome 1 ${intp.isInitializeComplete}" ) - - if (!intp.isInitializeComplete) { - intp.initializeSynchronous() - } - - echo(s"DB printWelcome 2 ${intp.isInitializeComplete}" ) - import org.apache.spark.SPARK_VERSION echo("""Welcome to ____ __ diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index 6f696ed7d8625..4e63816402a10 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -34,14 +34,8 @@ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter, initializeSpa * See the discussion in Scala community https://github.com/scala/bug/issues/10913 for detail. */ override def initializeSynchronous(): Unit = { - // scalastyle:off println - println(s"DB initializeSynchronous ${isInitializeComplete}" ) - // scalastyle:on println - - if (!isInitializeComplete) { - super.initializeSynchronous() - initializeSpark() - } + super.initializeSynchronous() + initializeSpark() } override lazy val memberHandlers = new {