address review comments

Author: sunchao

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

@@ -351,9 +351,9 @@ private[spark] class Client(
     // SPARK-37205: this regex is used to grep a list of configurations and send them to YARN RM
     // for fetching delegation tokens. See YARN-5910 for more details.
     // The feature is only supported in Hadoop 3.x and up, hence the check below.
-    val regex = sparkConf.get(config.AM_SEND_TOKEN_CONF)
+    val regex = sparkConf.get(config.AM_TOKEN_CONF_REGEX)
     if (regex.nonEmpty && VersionUtils.isHadoop3) {
-      logInfo(s"Processing token conf (spark.yarn.am.sendTokenConf) with regex $regex")
+      logInfo(s"Processing token conf (spark.yarn.am.tokenConfRegex) with regex $regex")
       val dob = new DataOutputBuffer();
       val copy = new Configuration(false);
       copy.clear();

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala

@@ -78,16 +78,19 @@ package object config extends Logging {
       .booleanConf
       .createWithDefault(false)
 
-  private[spark] val AM_SEND_TOKEN_CONF =
-    ConfigBuilder("spark.yarn.am.sendTokenConf")
+  private[spark] val AM_TOKEN_CONF_REGEX =
+    ConfigBuilder("spark.yarn.am.tokenConfRegex")
       .doc("This config is only supported for Hadoop 3.x profile. The value of this config is a " +
         "regex expression used to grep a list of config entries from the job's configuration " +
         "file (e.g., hdfs-site.xml) and send to RM, which uses them when renewing delegation " +
         "tokens. A typical use case of this feature is to support delegation tokens in an " +
         "environment where a YARN cluster needs to talk to multiple downstream HDFS clusters, " +
         "where the YARN RM may not have configs (e.g., dfs.nameservices, dfs.ha.namenodes.*, " +
-        "dfs.namenode.rpc-address.*) to connect to these clusters. This config is very similar " +
-        "to 'mapreduce.job.send-token-conf'. Please check YARN-5910 for more details.")
+        "dfs.namenode.rpc-address.*) to connect to these clusters. In this scenario, Spark " +
+        "users can specify the config value to be " +
+        "'^dfs.nameservices$|^dfs.namenode.rpc-address.*$|^dfs.ha.namenodes.*$' to parse " +
+        "these HDFS configs from the job's local configuration files. This config is very " +
+        "similar to 'mapreduce.job.send-token-conf'. Please check YARN-5910 for more details.")
       .version("3.3.0")
       .stringConf
       .createOptional