diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index dc16493c87878..3e9096d681642 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -606,48 +606,6 @@ from the other deployment modes. See the [configuration page](configuration.html myIdentifier. Multiple annotations can be added by setting multiple configurations with this prefix. - - spark.kubernetes.driver.labels - (none) - - Deprecated. Use spark.kubernetes.driver.label. instead which supports = - and , characters in label values. - Custom labels that will be added to the driver pod. This should be a comma-separated list of label key-value pairs, - where each label is in the format key=value. Note that Spark also adds its own labels to the driver pod - for bookkeeping purposes. - - - - spark.kubernetes.driver.annotations - (none) - - Deprecated. Use spark.kubernetes.driver.annotation. instead which supports - = and , characters in annotation values. - Custom annotations that will be added to the driver pod. This should be a comma-separated list of label key-value - pairs, where each annotation is in the format key=value. - - - - spark.kubernetes.executor.labels - (none) - - Deprecated. Use spark.kubernetes.executor.label. instead which supports - = and , characters in label values. - Custom labels that will be added to the executor pods. This should be a comma-separated list of label key-value - pairs, where each label is in the format key=value. Note that Spark also adds its own labels to the - executor pods for bookkeeping purposes. - - - - spark.kubernetes.executor.annotations - (none) - - Deprecated. Use spark.kubernetes.executor.annotation. instead which supports - = and , characters in annotation values. - Custom annotations that will be added to the executor pods. This should be a comma-separated list of annotation - key-value pairs, where each annotation is in the format key=value. - - spark.kubernetes.driver.pod.name (none) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/ConfigurationUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/ConfigurationUtils.scala index 7645e54e46280..601533615ce9a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/ConfigurationUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/ConfigurationUtils.scala @@ -19,7 +19,6 @@ package org.apache.spark.deploy.k8s import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.internal.Logging -import org.apache.spark.internal.config.OptionalConfigEntry object ConfigurationUtils extends Logging { def parseKeyValuePairs( @@ -41,31 +40,6 @@ object ConfigurationUtils extends Logging { }).getOrElse(Map.empty[String, String]) } - def combinePrefixedKeyValuePairsWithDeprecatedConf( - sparkConf: SparkConf, - prefix: String, - deprecatedConf: OptionalConfigEntry[String], - configType: String): Map[String, String] = { - val deprecatedKeyValuePairsString = sparkConf.get(deprecatedConf) - deprecatedKeyValuePairsString.foreach { _ => - logWarning(s"Configuration with key ${deprecatedConf.key} is deprecated. Use" + - s" configurations with prefix $prefix instead.") - } - val fromDeprecated = parseKeyValuePairs( - deprecatedKeyValuePairsString, - deprecatedConf.key, - configType) - val fromPrefix = sparkConf.getAllWithPrefix(prefix) - val combined = fromDeprecated.toSeq ++ fromPrefix - combined.groupBy(_._1).foreach { - case (key, values) => - require(values.size == 1, - s"Cannot have multiple values for a given $configType key, got key $key with" + - s" values $values") - } - combined.toMap - } - def parsePrefixedKeyValuePairs( sparkConf: SparkConf, prefix: String, diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/config.scala index ebabed956365d..fc08c0ad42f82 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/config.scala @@ -115,43 +115,12 @@ package object config extends Logging { private[spark] val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label." private[spark] val KUBERNETES_DRIVER_ANNOTATION_PREFIX = "spark.kubernetes.driver.annotation." + private[spark] val KUBERNETES_EXECUTOR_LABEL_PREFIX = "spark.kubernetes.executor.label." private[spark] val KUBERNETES_EXECUTOR_ANNOTATION_PREFIX = "spark.kubernetes.executor.annotation." - private[spark] val KUBERNETES_DRIVER_LABELS = - ConfigBuilder("spark.kubernetes.driver.labels") - .doc("Custom labels that will be added to the driver pod. This should be a comma-separated" + - " list of label key-value pairs, where each label is in the format key=value. Note that" + - " Spark also adds its own labels to the driver pod for bookkeeping purposes.") - .stringConf - .createOptional - private[spark] val KUBERNETES_DRIVER_ENV_KEY = "spark.kubernetes.driverEnv." - private[spark] val KUBERNETES_DRIVER_ANNOTATIONS = - ConfigBuilder("spark.kubernetes.driver.annotations") - .doc("Custom annotations that will be added to the driver pod. This should be a" + - " comma-separated list of annotation key-value pairs, where each annotation is in the" + - " format key=value.") - .stringConf - .createOptional - - private[spark] val KUBERNETES_EXECUTOR_LABELS = - ConfigBuilder("spark.kubernetes.executor.labels") - .doc("Custom labels that will be added to the executor pods. This should be a" + - " comma-separated list of label key-value pairs, where each label is in the format" + - " key=value.") - .stringConf - .createOptional - - private[spark] val KUBERNETES_EXECUTOR_ANNOTATIONS = - ConfigBuilder("spark.kubernetes.executor.annotations") - .doc("Custom annotations that will be added to the executor pods. This should be a" + - " comma-separated list of annotation key-value pairs, where each annotation is in the" + - " format key=value.") - .stringConf - .createOptional - private[spark] val KUBERNETES_DRIVER_SECRETS_PREFIX = "spark.kubernetes.driver.secrets." private[spark] val KUBERNETES_EXECUTOR_SECRETS_PREFIX = "spark.kubernetes.executor.secrets." diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigurationStepsOrchestrator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigurationStepsOrchestrator.scala index 17e4a18a07d47..fd251637ce210 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigurationStepsOrchestrator.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigurationStepsOrchestrator.scala @@ -72,10 +72,9 @@ private[spark] class DriverConfigurationStepsOrchestrator( .getOrElse(Array.empty[String]) ++ additionalMainAppPythonFile.toSeq ++ additionalPythonFiles - val driverCustomLabels = ConfigurationUtils.combinePrefixedKeyValuePairsWithDeprecatedConf( + val driverCustomLabels = ConfigurationUtils.parsePrefixedKeyValuePairs( submissionSparkConf, KUBERNETES_DRIVER_LABEL_PREFIX, - KUBERNETES_DRIVER_LABELS, "label") require(!driverCustomLabels.contains(SPARK_APP_ID_LABEL), s"Label with key " + s" $SPARK_APP_ID_LABEL is not allowed as it is reserved for Spark bookkeeping" + @@ -124,7 +123,7 @@ private[spark] class DriverConfigurationStepsOrchestrator( // Then, indicate to the outer block that the init-container should not handle // those local files simply by filtering them out. val sparkFilesWithoutLocal = KubernetesFileUtils.getNonSubmitterLocalFiles(sparkFiles) - val smallFilesSecretName = s"${kubernetesAppId}-submitted-files" + val smallFilesSecretName = s"$kubernetesAppId-submitted-files" val mountSmallFilesBootstrap = new MountSmallFilesBootstrapImpl( smallFilesSecretName, MOUNTED_SMALL_FILES_SECRET_MOUNT_PATH) val mountSmallLocalFilesStep = new MountSmallLocalFilesStep( diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStep.scala index 0eb9ee2563045..563662ef80d11 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStep.scala @@ -67,10 +67,9 @@ private[spark] class BaseDriverConfigurationStep( .build() } val driverCustomAnnotations = ConfigurationUtils - .combinePrefixedKeyValuePairsWithDeprecatedConf( + .parsePrefixedKeyValuePairs( submissionSparkConf, KUBERNETES_DRIVER_ANNOTATION_PREFIX, - KUBERNETES_DRIVER_ANNOTATIONS, "annotation") require(!driverCustomAnnotations.contains(SPARK_APP_NAME_ANNOTATION), s"Annotation with key $SPARK_APP_NAME_ANNOTATION is not allowed as it is reserved for" + diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala index f39ee1b6710d4..6f4ba1c8b888f 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala @@ -19,7 +19,6 @@ package org.apache.spark.scheduler.cluster.k8s import scala.collection.JavaConverters._ import io.fabric8.kubernetes.api.model.{ContainerBuilder, ContainerPortBuilder, EnvVar, EnvVarBuilder, EnvVarSourceBuilder, Pod, PodBuilder, QuantityBuilder} -import org.apache.commons.io.FilenameUtils import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.deploy.k8s.{ConfigurationUtils, InitContainerResourceStagingServerSecretPlugin, PodWithDetachedInitContainer, SparkPodInitContainerBootstrap} @@ -56,10 +55,9 @@ private[spark] class ExecutorPodFactoryImpl( org.apache.spark.internal.config.EXECUTOR_CLASS_PATH) private val executorJarsDownloadDir = sparkConf.get(INIT_CONTAINER_JARS_DOWNLOAD_LOCATION) - private val executorLabels = ConfigurationUtils.combinePrefixedKeyValuePairsWithDeprecatedConf( + private val executorLabels = ConfigurationUtils.parsePrefixedKeyValuePairs( sparkConf, KUBERNETES_EXECUTOR_LABEL_PREFIX, - KUBERNETES_EXECUTOR_LABELS, "executor label") require( !executorLabels.contains(SPARK_APP_ID_LABEL), @@ -70,10 +68,9 @@ private[spark] class ExecutorPodFactoryImpl( s" Spark.") private val executorAnnotations = - ConfigurationUtils.combinePrefixedKeyValuePairsWithDeprecatedConf( + ConfigurationUtils.parsePrefixedKeyValuePairs( sparkConf, KUBERNETES_EXECUTOR_ANNOTATION_PREFIX, - KUBERNETES_EXECUTOR_ANNOTATIONS, "executor annotation") private val nodeSelector = ConfigurationUtils.parsePrefixedKeyValuePairs( diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStepSuite.scala index 1441287244e0a..330d8e87a9fe9 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/BaseDriverConfigurationStepSuite.scala @@ -34,8 +34,6 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite { private val APP_ARGS = Array("arg1", "arg2") private val CUSTOM_ANNOTATION_KEY = "customAnnotation" private val CUSTOM_ANNOTATION_VALUE = "customAnnotationValue" - private val DEPRECATED_CUSTOM_ANNOTATION_KEY = "customAnnotationDeprecated" - private val DEPRECATED_CUSTOM_ANNOTATION_VALUE = "customAnnotationDeprecatedValue" private val DRIVER_CUSTOM_ENV_KEY1 = "customDriverEnv1" private val DRIVER_CUSTOM_ENV_KEY2 = "customDriverEnv2" @@ -49,8 +47,6 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite { .set(KUBERNETES_DRIVER_MEMORY_OVERHEAD, 200L) .set(DRIVER_DOCKER_IMAGE, "spark-driver:latest") .set(s"spark.kubernetes.driver.annotation.$CUSTOM_ANNOTATION_KEY", CUSTOM_ANNOTATION_VALUE) - .set("spark.kubernetes.driver.annotations", - s"$DEPRECATED_CUSTOM_ANNOTATION_KEY=$DEPRECATED_CUSTOM_ANNOTATION_VALUE") .set(s"$KUBERNETES_DRIVER_ENV_KEY$DRIVER_CUSTOM_ENV_KEY1", "customDriverEnv1") .set(s"$KUBERNETES_DRIVER_ENV_KEY$DRIVER_CUSTOM_ENV_KEY2", "customDriverEnv2") @@ -98,7 +94,6 @@ private[spark] class BaseDriverConfigurationStepSuite extends SparkFunSuite { assert(driverPodMetadata.getLabels.asScala === DRIVER_LABELS) val expectedAnnotations = Map( CUSTOM_ANNOTATION_KEY -> CUSTOM_ANNOTATION_VALUE, - DEPRECATED_CUSTOM_ANNOTATION_KEY -> DEPRECATED_CUSTOM_ANNOTATION_VALUE, SPARK_APP_NAME_ANNOTATION -> APP_NAME) assert(driverPodMetadata.getAnnotations.asScala === expectedAnnotations) assert(preparedDriverSpec.driverPod.getSpec.getRestartPolicy === "Never") diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initContainerBootstrapStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/InitContainerBootstrapStepSuite.scala similarity index 100% rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initContainerBootstrapStepSuite.scala rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/InitContainerBootstrapStepSuite.scala diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initcontainer/InitContainerConfigurationStepsOrchestratorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initcontainer/InitContainerConfigurationStepsOrchestratorSuite.scala index f29d12a9f26cc..3810a324f99b5 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initcontainer/InitContainerConfigurationStepsOrchestratorSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/submitsteps/initcontainer/InitContainerConfigurationStepsOrchestratorSuite.scala @@ -46,7 +46,6 @@ class InitContainerConfigurationStepsOrchestratorSuite extends SparkFunSuite { test ("error thrown if local jars provided without resource staging server") { val sparkConf = new SparkConf(true) - .set(KUBERNETES_DRIVER_LABELS, s"$DEPRECATED_CUSTOM_LABEL_KEY=$DEPRECATED_CUSTOM_LABEL_VALUE") .set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE) assert(sparkConf.get(RESOURCE_STAGING_SERVER_URI).isEmpty) @@ -72,7 +71,6 @@ class InitContainerConfigurationStepsOrchestratorSuite extends SparkFunSuite { test ("error not thrown with non-local jars and resource staging server provided") { val sparkConf = new SparkConf(true) - .set(KUBERNETES_DRIVER_LABELS, s"$DEPRECATED_CUSTOM_LABEL_KEY=$DEPRECATED_CUSTOM_LABEL_VALUE") .set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE) .set(RESOURCE_STAGING_SERVER_URI, STAGING_SERVER_URI) @@ -97,7 +95,6 @@ class InitContainerConfigurationStepsOrchestratorSuite extends SparkFunSuite { test ("error not thrown with non-local jars and no resource staging server provided") { val sparkConf = new SparkConf(true) - .set(KUBERNETES_DRIVER_LABELS, s"$DEPRECATED_CUSTOM_LABEL_KEY=$DEPRECATED_CUSTOM_LABEL_VALUE") .set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE) val orchestrator = new InitContainerConfigurationStepsOrchestrator( @@ -120,7 +117,6 @@ class InitContainerConfigurationStepsOrchestratorSuite extends SparkFunSuite { test ("including step to contact resource staging server") { val sparkConf = new SparkConf(true) - .set(KUBERNETES_DRIVER_LABELS, s"$DEPRECATED_CUSTOM_LABEL_KEY=$DEPRECATED_CUSTOM_LABEL_VALUE") .set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE) .set(RESOURCE_STAGING_SERVER_URI, STAGING_SERVER_URI) @@ -145,7 +141,6 @@ class InitContainerConfigurationStepsOrchestratorSuite extends SparkFunSuite { test ("not including steps because no contact to resource staging server") { val sparkConf = new SparkConf(true) - .set(KUBERNETES_DRIVER_LABELS, s"$DEPRECATED_CUSTOM_LABEL_KEY=$DEPRECATED_CUSTOM_LABEL_VALUE") .set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE) val orchestrator = new InitContainerConfigurationStepsOrchestrator( diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 0d052626d73d0..d6e12df9d1ba2 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -64,7 +64,7 @@ private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfter { sparkConf = kubernetesTestComponents.newSparkConf() .set(INIT_CONTAINER_DOCKER_IMAGE, s"spark-init:latest") .set(DRIVER_DOCKER_IMAGE, s"spark-driver:latest") - .set(KUBERNETES_DRIVER_LABELS, s"spark-app-locator=$APP_LOCATOR_LABEL") + .set(s"${KUBERNETES_DRIVER_LABEL_PREFIX}spark-app-locator", APP_LOCATOR_LABEL) kubernetesTestComponents.createNamespace() }