Skip to content

Commit b77c19b

Browse files
Marcelo Vanzinpwendell
authored andcommitted
Fix issue in ReplSuite with hadoop-provided profile.
When building the assembly with the maven "hadoop-provided" profile, the executors were failing to come up because Hadoop classes were not found in the classpath anymore; so add them explicitly to the classpath using spark.executor.extraClassPath. This is only needed for the local-cluster mode, but doesn't affect other tests, so it's added for all of them to keep the code simpler. Author: Marcelo Vanzin <[email protected]> Closes apache#781 from vanzin/repl-test-fix and squashes the following commits: 4f0a3b0 [Marcelo Vanzin] Fix issue in ReplSuite with hadoop-provided profile.
1 parent abea2d4 commit b77c19b

File tree

1 file changed

+13
-1
lines changed

1 file changed

+13
-1
lines changed

repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ import org.apache.spark.util.Utils
3232
class ReplSuite extends FunSuite {
3333

3434
def runInterpreter(master: String, input: String): String = {
35+
val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
36+
3537
val in = new BufferedReader(new StringReader(input + "\n"))
3638
val out = new StringWriter()
3739
val cl = getClass.getClassLoader
@@ -44,13 +46,23 @@ class ReplSuite extends FunSuite {
4446
}
4547
}
4648
}
49+
val classpath = paths.mkString(File.pathSeparator)
50+
51+
val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
52+
System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
53+
4754
val interp = new SparkILoop(in, new PrintWriter(out), master)
4855
org.apache.spark.repl.Main.interp = interp
49-
interp.process(Array("-classpath", paths.mkString(File.pathSeparator)))
56+
interp.process(Array("-classpath", classpath))
5057
org.apache.spark.repl.Main.interp = null
5158
if (interp.sparkContext != null) {
5259
interp.sparkContext.stop()
5360
}
61+
if (oldExecutorClasspath != null) {
62+
System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
63+
} else {
64+
System.clearProperty(CONF_EXECUTOR_CLASSPATH)
65+
}
5466
return out.toString
5567
}
5668

0 commit comments

Comments
 (0)