diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala index 930588fb0077..d6b1da39bcbb 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala @@ -28,6 +28,7 @@ import scala.jdk.CollectionConverters._ import io.fabric8.kubernetes.api.model.{ConfigMap, ConfigMapBuilder, KeyToPath} import org.apache.spark.SparkConf +import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} import org.apache.spark.deploy.k8s.{Config, Constants, KubernetesUtils} import org.apache.spark.deploy.k8s.Config.{KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH, KUBERNETES_NAMESPACE} import org.apache.spark.deploy.k8s.Constants.ENV_SPARK_CONF_DIR @@ -35,16 +36,26 @@ import org.apache.spark.internal.{Logging, MDC} import org.apache.spark.internal.LogKeys.{CONFIG, PATH, PATHS} import org.apache.spark.util.ArrayImplicits._ -private[spark] object KubernetesClientUtils extends Logging { +/** + * :: DeveloperApi :: + * + * A utility class used for K8s operations internally and Spark K8s operator. + */ +@Unstable +@DeveloperApi +object KubernetesClientUtils extends Logging { // Config map name can be KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH chars at max. + @Since("3.3.0") def configMapName(prefix: String): String = { val suffix = "-conf-map" s"${prefix.take(KUBERNETES_DNS_SUBDOMAIN_NAME_MAX_LENGTH - suffix.length)}$suffix" } + @Since("3.1.0") val configMapNameExecutor: String = configMapName(s"spark-exec-${KubernetesUtils.uniqueID()}") + @Since("3.1.0") val configMapNameDriver: String = configMapName(s"spark-drv-${KubernetesUtils.uniqueID()}") private def buildStringFromPropertiesMap(configMapName: String, @@ -62,6 +73,7 @@ private[spark] object KubernetesClientUtils extends Logging { /** * Build, file -> 'file's content' map of all the selected files in SPARK_CONF_DIR. */ + @Since("3.1.1") def buildSparkConfDirFilesMap( configMapName: String, sparkConf: SparkConf, @@ -77,6 +89,7 @@ private[spark] object KubernetesClientUtils extends Logging { } } + @Since("3.1.0") def buildKeyToPathObjects(confFilesMap: Map[String, String]): Seq[KeyToPath] = { confFilesMap.map { case (fileName: String, _: String) => @@ -89,6 +102,7 @@ private[spark] object KubernetesClientUtils extends Logging { * Build a Config Map that will hold the content for environment variable SPARK_CONF_DIR * on remote pods. */ + @Since("3.1.0") def buildConfigMap(configMapName: String, confFileMap: Map[String, String], withLabels: Map[String, String] = Map()): ConfigMap = { val configMapNameSpace =