diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 7a69f56ca4d4b..4788b8b14caa3 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -300,7 +300,8 @@ class SparkContext(config: SparkConf) extends Logging { // Create and start the scheduler private[spark] var taskScheduler = SparkContext.createTaskScheduler(this, master) - @volatile private[spark] var dagScheduler: DAGScheduler = new DAGScheduler(this) + @volatile private[spark] var dagSchedulerStopped = false + private val dagScheduler: DAGScheduler = new DAGScheduler(this) // start TaskScheduler after taskScheduler sets DAGScheduler reference in DAGScheduler's // constructor @@ -958,12 +959,13 @@ class SparkContext(config: SparkConf) extends Logging { ui.stop() // Do this only if not stopped already - best case effort. // prevent NPE if stopped more than once. - val dagSchedulerCopy = dagScheduler - dagScheduler = null - if (dagSchedulerCopy != null) { + if (dagSchedulerStopped) { + logInfo("SparkContext already stopped") + } else { metadataCleaner.cancel() cleaner.foreach(_.stop()) - dagSchedulerCopy.stop() + dagSchedulerStopped = true + dagScheduler.stop() taskScheduler = null // TODO: Cache.stop()? env.stop() @@ -973,8 +975,6 @@ class SparkContext(config: SparkConf) extends Logging { listenerBus.stop() eventLogger.foreach(_.stop()) logInfo("Successfully stopped SparkContext") - } else { - logInfo("SparkContext already stopped") } } diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala index d172dd1ac8e1b..9f7d8b222d181 100644 --- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala +++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala @@ -539,7 +539,7 @@ class DAGSchedulerSuite extends TestKit(ActorSystem("DAGSchedulerSuite")) with F watch(child) child ! "hi" expectMsgPF(){ case Terminated(child) => () } - assert(scheduler.sc.dagScheduler === null) + assert(scheduler.sc.dagSchedulerStopped === true) } /**