Skip to content

Commit 8df6b09

Browse files
author
Marcelo Vanzin
committed
Make sure py4j gateway is stopped.
py4j uses non-daemon threads internally, so if it's not explicitly stopped, it will prevent the process from exiting now that System.exit() is not being used.
1 parent 2bb2a8a commit 8df6b09

File tree

1 file changed

+17
-6
lines changed

1 file changed

+17
-6
lines changed

core/src/main/scala/org/apache/spark/deploy/PythonRunner.scala

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,14 @@ object PythonRunner {
4747
// Launch a Py4J gateway server for the process to connect to; this will let it see our
4848
// Java system properties and such
4949
val gatewayServer = new py4j.GatewayServer(null, 0)
50-
gatewayServer.start()
50+
val thread = new Thread(new Runnable() {
51+
override def run(): Unit = Utils.logUncaughtExceptions {
52+
gatewayServer.start()
53+
}
54+
})
55+
thread.setName("py4j-gateway")
56+
thread.setDaemon(true)
57+
thread.start()
5158

5259
// Build up a PYTHONPATH that includes the Spark assembly JAR (where this class is), the
5360
// python directories in SPARK_HOME (if set), and any files in the pyFiles argument
@@ -65,13 +72,17 @@ object PythonRunner {
6572
env.put("PYTHONUNBUFFERED", "YES") // value is needed to be set to a non-empty string
6673
env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort)
6774
builder.redirectErrorStream(true) // Ugly but needed for stdout and stderr to synchronize
68-
val process = builder.start()
75+
try {
76+
val process = builder.start()
6977

70-
new RedirectThread(process.getInputStream, System.out, "redirect output").start()
78+
new RedirectThread(process.getInputStream, System.out, "redirect output").start()
7179

72-
val exitCode = process.waitFor()
73-
if (exitCode != 0) {
74-
throw new SparkUserAppException(exitCode)
80+
val exitCode = process.waitFor()
81+
if (exitCode != 0) {
82+
throw new SparkUserAppException(exitCode)
83+
}
84+
} finally {
85+
gatewayServer.shutdown()
7586
}
7687
}
7788

0 commit comments

Comments
 (0)