From 9fb83468737ddd26b1eaccc691020cbf0d7fbe11 Mon Sep 17 00:00:00 2001 From: Marcelo Vanzin Date: Wed, 13 Feb 2019 17:05:52 -0800 Subject: [PATCH] [SPARK-24894][k8s] Make sure valid host names are created for executors. Since the host name is derived from the app name, which can contain arbitrary characters, it needs to be sanitized so that only valid characters are allowed. On top of that, take extra care that truncation doesn't leave characters that are valid except at the start of a host name. --- .../k8s/features/BasicExecutorFeatureStep.scala | 5 +++++ .../k8s/features/BasicExecutorFeatureStepSuite.scala | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala index 6c3a6b39fa5cb..af162839fef70 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala @@ -83,6 +83,11 @@ private[spark] class BasicExecutorFeatureStep( // name as the hostname. This preserves uniqueness since the end of name contains // executorId val hostname = name.substring(Math.max(0, name.length - 63)) + // Remove non-word characters from the start of the hostname + .replaceAll("^[^\\w]+", "") + // Replace dangerous characters in the remaining string with a safe alternative. + .replaceAll("[^\\w-]+", "_") + val executorMemoryQuantity = new QuantityBuilder(false) .withAmount(s"${executorMemoryTotal}Mi") .build() diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala index 36bfb7d41ec39..cb4cccc063031 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala @@ -22,6 +22,7 @@ import java.nio.file.Files import scala.collection.JavaConverters._ +import com.google.common.net.InternetDomainName import io.fabric8.kubernetes.api.model._ import org.scalatest.BeforeAndAfter @@ -124,6 +125,16 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter { assert(step.configurePod(SparkPod.initialPod()).pod.getSpec.getHostname.length === 63) } + test("hostname truncation generates valid host names") { + val invalidPrefix = "abcdef-*_/[]{}+==.,;'\"-----------------------------------------------" + + baseConf.set(KUBERNETES_EXECUTOR_POD_NAME_PREFIX, invalidPrefix) + val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf)) + val hostname = step.configurePod(SparkPod.initialPod()).pod.getSpec().getHostname() + assert(hostname.length <= 63) + assert(InternetDomainName.isValid(hostname)) + } + test("classpath and extra java options get translated into environment variables") { baseConf.set(config.EXECUTOR_JAVA_OPTIONS, "foo=bar") baseConf.set(config.EXECUTOR_CLASS_PATH, "bar=baz")