From 13d40d7533ac73d6fb82706a5ef1d19c9272c0e4 Mon Sep 17 00:00:00 2001 From: saturday_s Date: Wed, 7 Jun 2017 18:09:05 +0900 Subject: [PATCH 1/2] Not to read `spark.yarn.credentials.file` from checkpoint. --- .../src/main/scala/org/apache/spark/streaming/Checkpoint.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala index 5cbad8bf3ce6e..8feac12423f27 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala @@ -55,6 +55,7 @@ class Checkpoint(ssc: StreamingContext, val checkpointTime: Time) "spark.master", "spark.yarn.keytab", "spark.yarn.principal", + "spark.yarn.credentials.file", "spark.ui.filters") val newSparkConf = new SparkConf(loadDefaults = false).setAll(sparkConfPairs) From 2c50858be9fd422d32d5648b651a4b26ba3f8728 Mon Sep 17 00:00:00 2001 From: saturday_s Date: Mon, 12 Jun 2017 13:05:00 +0900 Subject: [PATCH 2/2] Not to read renewalTime and updateTime. --- .../src/main/scala/org/apache/spark/streaming/Checkpoint.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala index 8feac12423f27..b8c780db07c98 100644 --- a/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala +++ b/streaming/src/main/scala/org/apache/spark/streaming/Checkpoint.scala @@ -56,6 +56,8 @@ class Checkpoint(ssc: StreamingContext, val checkpointTime: Time) "spark.yarn.keytab", "spark.yarn.principal", "spark.yarn.credentials.file", + "spark.yarn.credentials.renewalTime", + "spark.yarn.credentials.updateTime", "spark.ui.filters") val newSparkConf = new SparkConf(loadDefaults = false).setAll(sparkConfPairs)