From 34ac2897bf1320aad24f7d1a87812335db1e13ca Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Tue, 29 May 2018 11:03:40 -0700 Subject: [PATCH 1/8] Upgrade scala to 2.11.12 and 2.12.6 Update LICENSE Update JLine update scala-parser-combinators Error handling Fix compilation temp Revert hack remove hack --- LICENSE | 12 +- dev/deps/spark-deps-hadoop-2.6 | 10 +- dev/deps/spark-deps-hadoop-2.7 | 10 +- dev/deps/spark-deps-hadoop-3.1 | 10 +- pom.xml | 8 +- .../org/apache/spark/repl/SparkILoop.scala | 145 ++++++++++++++++-- .../spark/repl/SparkILoopInterpreter.scala | 3 +- 7 files changed, 157 insertions(+), 41 deletions(-) diff --git a/LICENSE b/LICENSE index 820f14dbdeed..d3d920e6f2a0 100644 --- a/LICENSE +++ b/LICENSE @@ -242,18 +242,18 @@ The text of each license is also included at licenses/LICENSE-[project].txt. (BSD licence) ANTLR ST4 4.0.4 (org.antlr:ST4:4.0.4 - http://www.stringtemplate.org) (BSD licence) ANTLR StringTemplate (org.antlr:stringtemplate:3.2.1 - http://www.stringtemplate.org) (BSD License) Javolution (javolution:javolution:5.5.1 - http://javolution.org) - (BSD) JLine (jline:jline:0.9.94 - http://jline.sourceforge.net) + (BSD) JLine (jline:jline:2.14.3 - https://github.com/jline/jline2) (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.3 - http://paranamer.codehaus.org/paranamer) (BSD) ParaNamer Core (com.thoughtworks.paranamer:paranamer:2.6 - http://paranamer.codehaus.org/paranamer) (BSD 3 Clause) Scala (http://www.scala-lang.org/download/#License) (Interpreter classes (all .scala files in repl/src/main/scala except for Main.Scala, SparkHelper.scala and ExecutorClassLoader.scala), and for SerializableMapWrapper in JavaUtils.scala) - (BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scala Library (org.scala-lang:scala-library:2.11.8 - http://www.scala-lang.org/) - (BSD-like) Scalap (org.scala-lang:scalap:2.11.8 - http://www.scala-lang.org/) + (BSD-like) Scala Actors library (org.scala-lang:scala-actors:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Compiler (org.scala-lang:scala-compiler:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Compiler (org.scala-lang:scala-reflect:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scala Library (org.scala-lang:scala-library:2.11.12 - http://www.scala-lang.org/) + (BSD-like) Scalap (org.scala-lang:scalap:2.11.12 - http://www.scala-lang.org/) (BSD-style) scalacheck (org.scalacheck:scalacheck_2.11:1.10.0 - http://www.scalacheck.org) (BSD-style) spire (org.spire-math:spire_2.11:0.7.1 - http://spire-math.org) (BSD-style) spire-macros (org.spire-math:spire-macros_2.11:0.7.1 - http://spire-math.org) diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 723180a14feb..96e9c27210d0 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-6.1.26.jar jetty-util-6.1.26.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -172,10 +172,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar -scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar +scala-parser-combinators_2.11-1.1.0.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index ea08a001a1c9..4a6ee027ec35 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-6.1.26.jar jetty-util-6.1.26.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -173,10 +173,10 @@ parquet-jackson-1.10.0.jar protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar -scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar +scala-parser-combinators_2.11-1.1.0.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index da874026d7d1..e0b560c8ec71 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -122,7 +122,7 @@ jersey-server-2.22.2.jar jets3t-0.9.4.jar jetty-webapp-9.3.20.v20170531.jar jetty-xml-9.3.20.v20170531.jar -jline-2.12.1.jar +jline-2.14.3.jar joda-time-2.9.3.jar jodd-core-3.5.2.jar jpam-1.1.jar @@ -192,10 +192,10 @@ protobuf-java-2.5.0.jar py4j-0.10.7.jar pyrolite-4.13.jar re2j-1.1.jar -scala-compiler-2.11.8.jar -scala-library-2.11.8.jar -scala-parser-combinators_2.11-1.0.4.jar -scala-reflect-2.11.8.jar +scala-compiler-2.11.12.jar +scala-library-2.11.12.jar +scala-parser-combinators_2.11-1.1.0.jar +scala-reflect-2.11.12.jar scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar diff --git a/pom.xml b/pom.xml index 883c096ae1ae..4ccdfb34d3be 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ 3.4.1 3.2.2 - 2.11.8 + 2.11.12 2.11 1.9.13 2.6.7 @@ -740,13 +740,13 @@ org.scala-lang.modules scala-parser-combinators_${scala.binary.version} - 1.0.4 + 1.1.0 jline jline - 2.12.1 + 2.14.3 org.scalatest @@ -2748,7 +2748,7 @@ scala-2.12 - 2.12.4 + 2.12.6 2.12 diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index e69441a475e9..eef2ee0d1eb8 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -19,6 +19,14 @@ package org.apache.spark.repl import java.io.BufferedReader +import scala.concurrent.Future +import scala.reflect.classTag +import scala.reflect.internal.util.ScalaClassLoader.savingContextLoader +import scala.reflect.io.File +import scala.tools.nsc.interpreter.StdReplTags.tagOfIMain +import scala.tools.nsc.{GenericRunnerSettings, Properties} +import scala.tools.nsc.interpreter.{AbstractOrMissingHandler, IMain, NamedParam, SimpleReader, SplashLoop, SplashReader, isReplDebug, isReplPower, replProps} + // scalastyle:off println import scala.Predef.{println => _, _} // scalastyle:on println @@ -73,11 +81,15 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) "import org.apache.spark.sql.functions._" ) - def initializeSpark() { - intp.beQuietDuring { - savingReplayStack { // remove the commands from session history. - initializationCommands.foreach(processLine) + def initializeSpark(): Unit = { + if (!intp.reporter.hasErrors) { + // `savingReplayStack` removes the commands from session history. + savingReplayStack { + initializationCommands.foreach(intp quietRun _) } + } else { + throw new RuntimeException(s"Scala $versionString interpreter encountered " + + "errors during initialization") } } @@ -101,16 +113,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) /** Available commands */ override def commands: List[LoopCommand] = standardCommands - /** - * We override `loadFiles` because we need to initialize Spark *before* the REPL - * sees any files, so that the Spark context is visible in those files. This is a bit of a - * hack, but there isn't another hook available to us at this point. - */ - override def loadFiles(settings: Settings): Unit = { - initializeSpark() - super.loadFiles(settings) - } - override def resetCommand(line: String): Unit = { super.resetCommand(line) initializeSpark() @@ -122,6 +124,121 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) super.replay() } + override def process(settings: Settings): Boolean = savingContextLoader { + + def newReader = in0.fold(chooseReader(settings))(r => SimpleReader(r, out, interactive = true)) + + /** Reader to use before interpreter is online. */ + def preLoop = { + val sr = SplashReader(newReader) { r => + in = r + in.postInit() + } + in = sr + SplashLoop(sr, prompt) + } + + /* Actions to cram in parallel while collecting first user input at prompt. + * Run with output muted both from ILoop and from the intp reporter. + */ + def loopPostInit(): Unit = mumly { + // Bind intp somewhere out of the regular namespace where + // we can get at it in generated code. + intp.quietBind(NamedParam[IMain]("$intp", intp)(tagOfIMain, classTag[IMain])) + + // Auto-run code via some setting. + ( replProps.replAutorunCode.option + flatMap (f => File(f).safeSlurp()) + foreach (intp quietRun _) + ) + // power mode setup + if (isReplPower) { + replProps.power setValue true + unleashAndSetPhase() + asyncMessage(power.banner) + } + initializeSpark() + loadInitFiles() + // SI-7418 Now, and only now, can we enable TAB completion. + in.postInit() + } + def loadInitFiles(): Unit = settings match { + case settings: GenericRunnerSettings => + for (f <- settings.loadfiles.value) { + loadCommand(f) + addReplay(s":load $f") + } + for (f <- settings.pastefiles.value) { + pasteCommand(f) + addReplay(s":paste $f") + } + case _ => + } + // wait until after startup to enable noisy settings + def withSuppressedSettings[A](body: => A): A = { + val ss = this.settings + import ss._ + val noisy = List(Xprint, Ytyperdebug) + val noisesome = noisy.exists(!_.isDefault) + val current = (Xprint.value, Ytyperdebug.value) + if (isReplDebug || !noisesome) body + else { + this.settings.Xprint.value = List.empty + this.settings.Ytyperdebug.value = false + try body + finally { + Xprint.value = current._1 + Ytyperdebug.value = current._2 + intp.global.printTypings = current._2 + } + } + } + def startup(): String = withSuppressedSettings { + // while we go fire up the REPL + try { + // don't allow ancient sbt to hijack the reader + savingReader { + createInterpreter() + } + intp.initializeSynchronous() + globalFuture = Future successful true + if (intp.reporter.hasErrors) { + echo("Interpreter encountered errors during initialization!") + null + } else { + loopPostInit() + // starting + printWelcome() + + // let them start typing + val splash = preLoop + splash.start() + + val line = splash.line // what they typed in while they were waiting + if (line == null) { // they ^D + try out print Properties.shellInterruptedString + finally closeInterpreter() + } + line + } + } finally splash.stop() + } + + this.settings = settings + startup() match { + case null => false + case line => + try loop(line) match { + case LineResults.EOF => out print Properties.shellInterruptedString + case _ => + } + catch AbstractOrMissingHandler() + finally closeInterpreter() + true + } + } + + } object SparkILoop { diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index e736607a9a6b..386f421c57e5 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -21,8 +21,7 @@ import scala.collection.mutable import scala.tools.nsc.Settings import scala.tools.nsc.interpreter._ -class SparkILoopInterpreter(settings: Settings, out: JPrintWriter) extends IMain(settings, out) { - self => +class SparkILoopInterpreter(settings: Settings, out: JPrintWriter) extends IMain(settings, out) { self => override lazy val memberHandlers = new { val intp: self.type = self From 45c7c5b96eeb47c253f1551bfc780e4b321e25c5 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Thu, 28 Jun 2018 15:46:53 -0700 Subject: [PATCH 2/8] temp --- .../org/apache/spark/repl/SparkILoopInterpreter.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala index a0f67c5f880f..e736607a9a6b 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala @@ -24,10 +24,10 @@ import scala.tools.nsc.interpreter._ class SparkILoopInterpreter(settings: Settings, out: JPrintWriter) extends IMain(settings, out) { self => - override lazy val memberHandlers = new { - val intp: self.type = self - } with MemberHandlers { - import intp.global._ + override lazy val memberHandlers = new { + val intp: self.type = self + } with MemberHandlers { + import intp.global._ override def chooseHandler(member: intp.global.Tree): MemberHandler = member match { case member: Import => new SparkImportHandler(member) From e7d02a903438ec5dc3a6151402d1033bcf45c746 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Thu, 28 Jun 2018 17:16:49 -0700 Subject: [PATCH 3/8] update --- .../org/apache/spark/repl/SparkILoop.scala | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index b92772c75470..2dbcbea73821 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -19,19 +19,19 @@ package org.apache.spark.repl import java.io.BufferedReader +// scalastyle:off println +import scala.Predef.{println => _, _} +// scalastyle:on println import scala.concurrent.Future import scala.reflect.classTag import scala.reflect.internal.util.ScalaClassLoader.savingContextLoader import scala.reflect.io.File -import scala.tools.nsc.interpreter.StdReplTags.tagOfIMain import scala.tools.nsc.{GenericRunnerSettings, Properties} -import scala.tools.nsc.interpreter.{AbstractOrMissingHandler, IMain, NamedParam, SimpleReader, SplashLoop, SplashReader, isReplDebug, isReplPower, replProps} - -// scalastyle:off println -import scala.Predef.{println => _, _} -// scalastyle:on println import scala.tools.nsc.Settings -import scala.tools.nsc.interpreter.{ILoop, JPrintWriter} +import scala.tools.nsc.interpreter.{isReplDebug, isReplPower, replProps} +import scala.tools.nsc.interpreter.{AbstractOrMissingHandler, ILoop, IMain, JPrintWriter} +import scala.tools.nsc.interpreter.{NamedParam, SimpleReader, SplashLoop, SplashReader} +import scala.tools.nsc.interpreter.StdReplTags.tagOfIMain import scala.tools.nsc.util.stringFromStream import scala.util.Properties.{javaVersion, javaVmName, versionString} @@ -124,6 +124,20 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) super.replay() } + /** + * The following code is mostly a copy of `process` implementation in `ILoop.scala` in Scala + * + * In newer version of Scala, `printWelcome` is the first thing to be called. As a result, + * SparkUI URL information would be always shown after the welcome message. + * + * However, this is inconsistent to the existing version of Spark which will always + * show SparkUI URL first. + * + * The only way we can make it consistent will be duplicate the Scala code. + * + * We should remove this duplication once Scala provides a way to load our custom initialization + * code, and also customize the ordering of printing welcome message. + */ override def process(settings: Settings): Boolean = savingContextLoader { def newReader = in0.fold(chooseReader(settings))(r => SimpleReader(r, out, interactive = true)) @@ -207,9 +221,8 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) null } else { loopPostInit() - // starting - printWelcome() + printWelcome() // let them start typing val splash = preLoop splash.start() @@ -237,8 +250,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) true } } - - } object SparkILoop { From 3f71d20e5ecaacd5dec32beea2da580927697e81 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Tue, 10 Jul 2018 16:01:55 -0700 Subject: [PATCH 4/8] temp --- .../scala/org/apache/spark/repl/SparkILoop.scala | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 2dbcbea73821..2eef714b7bc0 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -44,7 +44,7 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) def this() = this(None, new JPrintWriter(Console.out, true)) override def createInterpreter(): Unit = { - intp = new SparkILoopInterpreter(settings, out, initializeSpark) + intp = new SparkILoopInterpreter(settings, out) } val initializationCommands: Seq[String] = Seq( @@ -166,12 +166,7 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) foreach (intp quietRun _) ) // power mode setup - if (isReplPower) { - replProps.power setValue true - unleashAndSetPhase() - asyncMessage(power.banner) - } - initializeSpark() + if (isReplPower) enablePowerMode(true) loadInitFiles() // SI-7418 Now, and only now, can we enable TAB completion. in.postInit() @@ -209,13 +204,17 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) } def startup(): String = withSuppressedSettings { // while we go fire up the REPL + val splash = preLoop + try { // don't allow ancient sbt to hijack the reader savingReader { createInterpreter() } intp.initializeSynchronous() - globalFuture = Future successful true + initializeSpark() + + // globalFuture = Future successful true if (intp.reporter.hasErrors) { echo("Interpreter encountered errors during initialization!") null @@ -224,7 +223,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) printWelcome() // let them start typing - val splash = preLoop splash.start() val line = splash.line // what they typed in while they were waiting From b1e28e82f0ad9b78e0d03f6cb0f27d7b097f5915 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Tue, 10 Jul 2018 18:14:36 -0700 Subject: [PATCH 5/8] working --- .../scala/org/apache/spark/repl/SparkILoop.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 2eef714b7bc0..2ada4f78c19d 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -19,6 +19,8 @@ package org.apache.spark.repl import java.io.BufferedReader +import scala.concurrent.Await + // scalastyle:off println import scala.Predef.{println => _, _} // scalastyle:on println @@ -167,6 +169,7 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) ) // power mode setup if (isReplPower) enablePowerMode(true) + initializeSpark() loadInitFiles() // SI-7418 Now, and only now, can we enable TAB completion. in.postInit() @@ -203,26 +206,28 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) } } def startup(): String = withSuppressedSettings { - // while we go fire up the REPL + // let them start typing val splash = preLoop + // while we go fire up the REPL try { // don't allow ancient sbt to hijack the reader savingReader { createInterpreter() } intp.initializeSynchronous() - initializeSpark() - // globalFuture = Future successful true + val field = classOf[ILoop].getDeclaredFields.filter(_.getName.contains("globalFuture")).head + field.setAccessible(true) + field.set(this, Future successful true) + + // val globalFuture: Future[Boolean] = if (intp.reporter.hasErrors) { echo("Interpreter encountered errors during initialization!") null } else { loopPostInit() - printWelcome() - // let them start typing splash.start() val line = splash.line // what they typed in while they were waiting From f0a76a37cd82b8a4f2614993f1a71915fd2a900c Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 11 Jul 2018 11:20:44 -0700 Subject: [PATCH 6/8] Remove message --- .../src/main/scala/org/apache/spark/repl/SparkILoop.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala index 2ada4f78c19d..94265267b1f9 100644 --- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -19,8 +19,6 @@ package org.apache.spark.repl import java.io.BufferedReader -import scala.concurrent.Await - // scalastyle:off println import scala.Predef.{println => _, _} // scalastyle:on println @@ -132,10 +130,10 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) * In newer version of Scala, `printWelcome` is the first thing to be called. As a result, * SparkUI URL information would be always shown after the welcome message. * - * However, this is inconsistent to the existing version of Spark which will always + * However, this is inconsistent compared with the existing version of Spark which will always * show SparkUI URL first. * - * The only way we can make it consistent will be duplicate the Scala code. + * The only way we can make it consistent will be duplicating the Scala code. * * We should remove this duplication once Scala provides a way to load our custom initialization * code, and also customize the ordering of printing welcome message. @@ -221,7 +219,6 @@ class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) field.setAccessible(true) field.set(this, Future successful true) - // val globalFuture: Future[Boolean] = if (intp.reporter.hasErrors) { echo("Interpreter encountered errors during initialization!") null From 8715fbeb0efdd591df0a5138393e8d2aa867ef8c Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Tue, 21 Aug 2018 15:26:46 -0700 Subject: [PATCH 7/8] consolidate code --- .../org/apache/spark/repl/SparkILoop.scala | 143 ------------------ .../apache/spark/repl/SparkExprTyper.scala | 0 .../org/apache/spark/repl/SparkILoop.scala | 0 .../spark/repl/SparkILoopInterpreter.scala | 0 4 files changed, 143 deletions(-) delete mode 100644 repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala rename repl/{scala-2.11 => }/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala (100%) rename repl/{scala-2.11 => }/src/main/scala/org/apache/spark/repl/SparkILoop.scala (100%) rename repl/{scala-2.11 => }/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala (100%) diff --git a/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala deleted file mode 100644 index ffb2e5f5db7e..000000000000 --- a/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.repl - -import java.io.BufferedReader - -import scala.tools.nsc.Settings -import scala.tools.nsc.interpreter.{ILoop, JPrintWriter} -import scala.tools.nsc.util.stringFromStream -import scala.util.Properties.{javaVersion, javaVmName, versionString} - -/** - * A Spark-specific interactive shell. - */ -class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) - extends ILoop(in0, out) { - def this(in0: BufferedReader, out: JPrintWriter) = this(Some(in0), out) - def this() = this(None, new JPrintWriter(Console.out, true)) - - val initializationCommands: Seq[String] = Seq( - """ - @transient val spark = if (org.apache.spark.repl.Main.sparkSession != null) { - org.apache.spark.repl.Main.sparkSession - } else { - org.apache.spark.repl.Main.createSparkSession() - } - @transient val sc = { - val _sc = spark.sparkContext - if (_sc.getConf.getBoolean("spark.ui.reverseProxy", false)) { - val proxyUrl = _sc.getConf.get("spark.ui.reverseProxyUrl", null) - if (proxyUrl != null) { - println( - s"Spark Context Web UI is available at ${proxyUrl}/proxy/${_sc.applicationId}") - } else { - println(s"Spark Context Web UI is available at Spark Master Public URL") - } - } else { - _sc.uiWebUrl.foreach { - webUrl => println(s"Spark context Web UI available at ${webUrl}") - } - } - println("Spark context available as 'sc' " + - s"(master = ${_sc.master}, app id = ${_sc.applicationId}).") - println("Spark session available as 'spark'.") - _sc - } - """, - "import org.apache.spark.SparkContext._", - "import spark.implicits._", - "import spark.sql", - "import org.apache.spark.sql.functions._" - ) - - def initializeSpark() { - intp.beQuietDuring { - savingReplayStack { // remove the commands from session history. - initializationCommands.foreach(command) - } - } - } - - /** Print a welcome message */ - override def printWelcome() { - import org.apache.spark.SPARK_VERSION - echo("""Welcome to - ____ __ - / __/__ ___ _____/ /__ - _\ \/ _ \/ _ `/ __/ '_/ - /___/ .__/\_,_/_/ /_/\_\ version %s - /_/ - """.format(SPARK_VERSION)) - val welcomeMsg = "Using Scala %s (%s, Java %s)".format( - versionString, javaVmName, javaVersion) - echo(welcomeMsg) - echo("Type in expressions to have them evaluated.") - echo("Type :help for more information.") - } - - /** Available commands */ - override def commands: List[LoopCommand] = standardCommands - - /** - * We override `createInterpreter` because we need to initialize Spark *before* the REPL - * sees any files, so that the Spark context is visible in those files. This is a bit of a - * hack, but there isn't another hook available to us at this point. - */ - override def createInterpreter(): Unit = { - super.createInterpreter() - initializeSpark() - } - - override def resetCommand(line: String): Unit = { - super.resetCommand(line) - initializeSpark() - echo("Note that after :reset, state of SparkSession and SparkContext is unchanged.") - } - - override def replay(): Unit = { - initializeSpark() - super.replay() - } - -} - -object SparkILoop { - - /** - * Creates an interpreter loop with default settings and feeds - * the given code to it as input. - */ - def run(code: String, sets: Settings = new Settings): String = { - import java.io.{ BufferedReader, StringReader, OutputStreamWriter } - - stringFromStream { ostream => - Console.withOut(ostream) { - val input = new BufferedReader(new StringReader(code)) - val output = new JPrintWriter(new OutputStreamWriter(ostream), true) - val repl = new SparkILoop(input, output) - - if (sets.classpath.isDefault) { - sets.classpath.value = sys.props("java.class.path") - } - repl process sets - } - } - } - def run(lines: List[String]): String = run(lines.map(_ + "\n").mkString) -} diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala b/repl/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala similarity index 100% rename from repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala rename to repl/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala similarity index 100% rename from repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala rename to repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala similarity index 100% rename from repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala rename to repl/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala From b8c18373a3be0c62d217ee76b4a5273c17f63ccf Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 22 Aug 2018 00:09:45 -0700 Subject: [PATCH 8/8] Revert "consolidate code" This reverts commit 8715fbeb0efdd591df0a5138393e8d2aa867ef8c. --- .../apache/spark/repl/SparkExprTyper.scala | 0 .../org/apache/spark/repl/SparkILoop.scala | 0 .../spark/repl/SparkILoopInterpreter.scala | 0 .../org/apache/spark/repl/SparkILoop.scala | 143 ++++++++++++++++++ 4 files changed, 143 insertions(+) rename repl/{ => scala-2.11}/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala (100%) rename repl/{ => scala-2.11}/src/main/scala/org/apache/spark/repl/SparkILoop.scala (100%) rename repl/{ => scala-2.11}/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala (100%) create mode 100644 repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala similarity index 100% rename from repl/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala rename to repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkExprTyper.scala diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala similarity index 100% rename from repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala rename to repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoop.scala diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala similarity index 100% rename from repl/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala rename to repl/scala-2.11/src/main/scala/org/apache/spark/repl/SparkILoopInterpreter.scala diff --git a/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala new file mode 100644 index 000000000000..ffb2e5f5db7e --- /dev/null +++ b/repl/scala-2.12/src/main/scala/org/apache/spark/repl/SparkILoop.scala @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.repl + +import java.io.BufferedReader + +import scala.tools.nsc.Settings +import scala.tools.nsc.interpreter.{ILoop, JPrintWriter} +import scala.tools.nsc.util.stringFromStream +import scala.util.Properties.{javaVersion, javaVmName, versionString} + +/** + * A Spark-specific interactive shell. + */ +class SparkILoop(in0: Option[BufferedReader], out: JPrintWriter) + extends ILoop(in0, out) { + def this(in0: BufferedReader, out: JPrintWriter) = this(Some(in0), out) + def this() = this(None, new JPrintWriter(Console.out, true)) + + val initializationCommands: Seq[String] = Seq( + """ + @transient val spark = if (org.apache.spark.repl.Main.sparkSession != null) { + org.apache.spark.repl.Main.sparkSession + } else { + org.apache.spark.repl.Main.createSparkSession() + } + @transient val sc = { + val _sc = spark.sparkContext + if (_sc.getConf.getBoolean("spark.ui.reverseProxy", false)) { + val proxyUrl = _sc.getConf.get("spark.ui.reverseProxyUrl", null) + if (proxyUrl != null) { + println( + s"Spark Context Web UI is available at ${proxyUrl}/proxy/${_sc.applicationId}") + } else { + println(s"Spark Context Web UI is available at Spark Master Public URL") + } + } else { + _sc.uiWebUrl.foreach { + webUrl => println(s"Spark context Web UI available at ${webUrl}") + } + } + println("Spark context available as 'sc' " + + s"(master = ${_sc.master}, app id = ${_sc.applicationId}).") + println("Spark session available as 'spark'.") + _sc + } + """, + "import org.apache.spark.SparkContext._", + "import spark.implicits._", + "import spark.sql", + "import org.apache.spark.sql.functions._" + ) + + def initializeSpark() { + intp.beQuietDuring { + savingReplayStack { // remove the commands from session history. + initializationCommands.foreach(command) + } + } + } + + /** Print a welcome message */ + override def printWelcome() { + import org.apache.spark.SPARK_VERSION + echo("""Welcome to + ____ __ + / __/__ ___ _____/ /__ + _\ \/ _ \/ _ `/ __/ '_/ + /___/ .__/\_,_/_/ /_/\_\ version %s + /_/ + """.format(SPARK_VERSION)) + val welcomeMsg = "Using Scala %s (%s, Java %s)".format( + versionString, javaVmName, javaVersion) + echo(welcomeMsg) + echo("Type in expressions to have them evaluated.") + echo("Type :help for more information.") + } + + /** Available commands */ + override def commands: List[LoopCommand] = standardCommands + + /** + * We override `createInterpreter` because we need to initialize Spark *before* the REPL + * sees any files, so that the Spark context is visible in those files. This is a bit of a + * hack, but there isn't another hook available to us at this point. + */ + override def createInterpreter(): Unit = { + super.createInterpreter() + initializeSpark() + } + + override def resetCommand(line: String): Unit = { + super.resetCommand(line) + initializeSpark() + echo("Note that after :reset, state of SparkSession and SparkContext is unchanged.") + } + + override def replay(): Unit = { + initializeSpark() + super.replay() + } + +} + +object SparkILoop { + + /** + * Creates an interpreter loop with default settings and feeds + * the given code to it as input. + */ + def run(code: String, sets: Settings = new Settings): String = { + import java.io.{ BufferedReader, StringReader, OutputStreamWriter } + + stringFromStream { ostream => + Console.withOut(ostream) { + val input = new BufferedReader(new StringReader(code)) + val output = new JPrintWriter(new OutputStreamWriter(ostream), true) + val repl = new SparkILoop(input, output) + + if (sets.classpath.isDefault) { + sets.classpath.value = sys.props("java.class.path") + } + repl process sets + } + } + } + def run(lines: List[String]): String = run(lines.map(_ + "\n").mkString) +}