Change the renewal interval config param. Fix a bunch of comments.

Author: harishreedharan

core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala

@@ -242,7 +242,10 @@ class SparkHadoopUtil extends Logging {
       fraction: Double,
       credentials: Credentials): Long = {
     val now = System.currentTimeMillis()
-    val renewalInterval = sparkConf.getLong("spark.yarn.renewal.interval", (24 hours).toMillis)
+
+    val renewalInterval =
+      sparkConf.getLong("spark.yarn.token.renewal.interval", (24 hours).toMillis)
+
     credentials.getAllTokens.filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND)
       .map { t =>
       val identifier = new DelegationTokenIdentifier()

core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala

@@ -192,8 +192,8 @@ private[spark] object CoarseGrainedExecutorBackend extends Logging {
       workerUrl.foreach { url =>
         env.rpcEnv.setupEndpoint("WorkerWatcher", new WorkerWatcher(env.rpcEnv, url))
       }
-      tokenUpdaterOption.foreach(_.stop())
       env.rpcEnv.awaitTermination()
+      tokenUpdaterOption.foreach(_.stop())
     }
   }
 

yarn/src/main/scala/org/apache/spark/deploy/yarn/AMDelegationTokenRenewer.scala

@@ -64,8 +64,9 @@ private[yarn] class AMDelegationTokenRenewer(
   /**
    * Schedule a login from the keytab and principal set using the --principal and --keytab
    * arguments to spark-submit. This login happens only when the credentials of the current user
-   * are about to expire. This method reads SPARK_PRINCIPAL and SPARK_KEYTAB from the environment
-   * to do the login. This method is a no-op in non-YARN mode.
+   * are about to expire. This method reads spark.yarn.principal and spark.yarn.keytab from
+   * SparkConf to do the login. This method is a no-op in non-YARN mode.
+   *
    */
   private[spark] def scheduleLoginFromKeytab(): Unit = {
     val principal = sparkConf.get("spark.yarn.principal")

yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

@@ -444,7 +444,7 @@ private[spark] class Client(
         "spark.yarn.credentials.file", new Path(stagingDirPath, credentialsFile).toString)
       logInfo(s"Credentials file set to: $credentialsFile")
       val renewalInterval = getTokenRenewalInterval(stagingDirPath)
-      sparkConf.set("spark.yarn.renewal.interval", renewalInterval.toString)
+      sparkConf.set("spark.yarn.token.renewal.interval", renewalInterval.toString)
     }
     // Set the environment variables to be passed on to the executors.
     distCacheMgr.setDistFilesEnv(env)