Skip to content

Commit f957796

Browse files
SaintBacchustdas
authored andcommitted
[SPARK-8820] [STREAMING] Add a configuration to set checkpoint dir.
Add a configuration to set checkpoint directory for convenience to user. [Jira Address](https://issues.apache.org/jira/browse/SPARK-8820) Author: huangzhaowei <[email protected]> Closes apache#7218 from SaintBacchus/SPARK-8820 and squashes the following commits: d49fe4b [huangzhaowei] Rename the configuration name 66ea47c [huangzhaowei] Add the unit test. dd0acc1 [huangzhaowei] [SPARK-8820][Streaming] Add a configuration to set checkpoint dir.
1 parent cc57d70 commit f957796

File tree

2 files changed

+11
-0
lines changed

2 files changed

+11
-0
lines changed

streaming/src/main/scala/org/apache/spark/streaming/StreamingContext.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,8 @@ class StreamingContext private[streaming] (
201201

202202
private var shutdownHookRef: AnyRef = _
203203

204+
conf.getOption("spark.streaming.checkpoint.directory").foreach(checkpoint)
205+
204206
/**
205207
* Return the associated Spark context
206208
*/

streaming/src/test/scala/org/apache/spark/streaming/StreamingContextSuite.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,15 @@ class StreamingContextSuite extends SparkFunSuite with BeforeAndAfter with Timeo
115115
assert(ssc.conf.getTimeAsSeconds("spark.cleaner.ttl", "-1") === 10)
116116
}
117117

118+
test("checkPoint from conf") {
119+
val checkpointDirectory = Utils.createTempDir().getAbsolutePath()
120+
121+
val myConf = SparkContext.updatedConf(new SparkConf(false), master, appName)
122+
myConf.set("spark.streaming.checkpoint.directory", checkpointDirectory)
123+
val ssc = new StreamingContext(myConf, batchDuration)
124+
assert(ssc.checkpointDir != null)
125+
}
126+
118127
test("state matching") {
119128
import StreamingContextState._
120129
assert(INITIALIZED === INITIALIZED)

0 commit comments

Comments
 (0)