From a1a7f59a525ce948df6be6af3745b9ae58d94cea Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Tue, 23 Oct 2018 14:11:16 +0100 Subject: [PATCH 1/7] [SPARK-25809][K8S] Docker for desktop K8S integration backend Adds a new K8S integration testing backend that allows for using the Kubernetes support in Docker for Desktop --- .../k8s/integrationtest/ProcessUtils.scala | 5 +- .../backend/IntegrationTestBackend.scala | 16 +++-- .../docker/DockerForDesktopBackend.scala | 68 +++++++++++++++++++ 3 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala index d8f3a6cec05c..78e0083b4716 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -28,7 +28,7 @@ object ProcessUtils extends Logging { * executeProcess is used to run a command and return the output if it * completes within timeout seconds. */ - def executeProcess(fullCommand: Array[String], timeout: Long): Seq[String] = { + def executeProcess(fullCommand: Array[String], timeout: Long, dumpErrors: Boolean = false): Seq[String] = { val pb = new ProcessBuilder().command(fullCommand: _*) pb.redirectErrorStream(true) val proc = pb.start() @@ -40,7 +40,8 @@ object ProcessUtils extends Logging { }) assert(proc.waitFor(timeout, TimeUnit.SECONDS), s"Timed out while executing ${fullCommand.mkString(" ")}") - assert(proc.exitValue == 0, s"Failed to execute ${fullCommand.mkString(" ")}") + assert(proc.exitValue == 0, + s"Failed to execute ${fullCommand.mkString(" ")}${if (dumpErrors) outputLines.mkString("\n")}") outputLines } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index 284712c6d250..e83a56a4b973 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.integrationtest.backend import io.fabric8.kubernetes.client.DefaultKubernetesClient - +import org.apache.spark.deploy.k8s.integrationtest.backend.docker.DockerForDesktopBackend import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend private[spark] trait IntegrationTestBackend { @@ -30,14 +30,16 @@ private[spark] trait IntegrationTestBackend { private[spark] object IntegrationTestBackendFactory { val deployModeConfigKey = "spark.kubernetes.test.deployMode" + val backendMinikube = "minikube" + val backendDockerForDesktop = "docker-for-desktop" + def getTestBackend: IntegrationTestBackend = { val deployMode = Option(System.getProperty(deployModeConfigKey)) - .getOrElse("minikube") - if (deployMode == "minikube") { - MinikubeTestBackend - } else { - throw new IllegalArgumentException( - "Invalid " + deployModeConfigKey + ": " + deployMode) + .getOrElse(backendMinikube) + deployMode match { + case `backendMinikube` => MinikubeTestBackend + case `backendDockerForDesktop` => DockerForDesktopBackend + case _ => throw new IllegalArgumentException("Invalid " + deployModeConfigKey + ": " + deployMode) } } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala new file mode 100644 index 000000000000..f7644c98fc6d --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala @@ -0,0 +1,68 @@ +package org.apache.spark.deploy.k8s.integrationtest.backend.docker + +import java.nio.file.Paths + +import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} +import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object DockerForDesktopBackend extends IntegrationTestBackend { + + private val KUBECTL_STARTUP_TIMEOUT_SECONDS = 15 + + private var defaultClient: DefaultKubernetesClient = _ + private var initialContext = "" + + private def getCurrentContext: String = { + val outputs = executeKubectl("config", "current-context") + assert(outputs.size == 1, "Unexpected amount of output from kubectl config current-context") + outputs.head + } + + private def setContext(context: String): Unit = { + val outputs = executeKubectl("config", "use-context", context) + assert(outputs.size == 1, "Unexpected amount of output from kubectl config use-context") + val errors = outputs.filter(_.startsWith("error")) + assert(errors.size == 0, s"Received errors from kubectl: ${errors.head}") + } + + override def initialize(): Unit = { + // Switch context if necessary + // TODO: If we were using Fabric 8 client 3.1.0 then we could + // instead just use the overload of autoConfigure() that takes the + // desired context avoiding the need to interact with kubectl at all + initialContext = getCurrentContext + if (!initialContext.equals("docker-for-desktop")) { + setContext("docker-for-desktop") + } + + // Auto-configure K8S client from K8S config file + System.setProperty(Config.KUBERNETES_AUTH_TRYKUBECONFIG_SYSTEM_PROPERTY, "true"); + val userHome = System.getProperty("user.home") + System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, + Option(System.getenv("KUBECONFIG")) + .getOrElse(Paths.get(userHome, ".kube", "config").toFile.getAbsolutePath)) + val config = Config.autoConfigure() + + defaultClient = new DefaultKubernetesClient(config) + } + + override def cleanUp(): Unit = { + super.cleanUp() + + // Reset users kubectl context appropriately if necessary + if (!initialContext.equals("docker-for-desktop")) { + setContext(initialContext) + } + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } + + private def executeKubectl(args: String*): Seq[String] = { + ProcessUtils.executeProcess( + Array("kubectl") ++ args, KUBECTL_STARTUP_TIMEOUT_SECONDS, true) + } + +} From f5f8487a77aebf20809b129703af9a9192e3ef74 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Wed, 24 Oct 2018 15:57:55 +0100 Subject: [PATCH 2/7] [SPARK-25809][K8S] Integration test improvements - Adds cloud and cloud-url backends - Improves logic in integration test scripts to cover new properties - Move constants into TestConstants - Use Config.autoConfigure() with context for simpler implementation without the need to shell out - Improve ReadMe with more detail on test backends, configuration and assumptions --- .../k8s/submit/LoggingPodStatusWatcher.scala | 3 - .../kubernetes/integration-tests/README.md | 159 +++++++++++++++++- .../dev/dev-run-integration-tests.sh | 10 ++ .../kubernetes/integration-tests/pom.xml | 10 ++ .../scripts/setup-integration-test-env.sh | 41 +++-- .../k8s/integrationtest/TestConstants.scala | 23 ++- .../deploy/k8s/integrationtest/Utils.scala | 32 ++++ .../backend/IntegrationTestBackend.scala | 20 +-- .../backend/cloud/CloudTestBackend.scala | 40 +++++ .../backend/cloud/KubeConfigBackend.scala | 58 +++++++ .../docker/DockerForDesktopBackend.scala | 83 +++------ 11 files changed, 382 insertions(+), 97 deletions(-) create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala index 79b55bc37afc..a2430c05e256 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala @@ -18,13 +18,10 @@ package org.apache.spark.deploy.k8s.submit import java.util.concurrent.{CountDownLatch, TimeUnit} -import scala.collection.JavaConverters._ - import io.fabric8.kubernetes.api.model.Pod import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher} import io.fabric8.kubernetes.client.Watcher.Action -import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.KubernetesUtils._ import org.apache.spark.internal.Logging import org.apache.spark.util.ThreadUtils diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index b3863e6b7d1a..de5241693eb5 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -13,15 +13,45 @@ The simplest way to run the integration tests is to install and run Minikube, th dev/dev-run-integration-tests.sh The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should -run with a minimum of 3 CPUs and 4G of memory: +run with a minimum of 4 CPUs and 6G of memory: - minikube start --cpus 3 --memory 4096 + minikube start --cpus 4 --memory 6144 You can download Minikube [here](https://github.com/kubernetes/minikube/releases). # Integration test customization -Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. +Configuration of the integration test runtime is done through passing different arguments to the test script. +The main useful options are outlined below. + +## Using a different backend + +The integration test backend i.e. the K8S cluster used for testing is controlled by the `--deploy-mode` option. By default this +is set to `minikube`, the available backends are their perquisites are as follows. + +### `minikube` + +Uses the local `minikube` cluster, this requires that `minikube` 0.23.0 or greater be installed and that it be allocated at least +4 CPUs and 6GB memory (some users have reported success with as few as 3 CPUs and 4GB memory). The tests will check if `minikube` +is started and abort early if it isn't currently running. + +### `docker-for-desktop` + +Since July 2018 Docker for Desktop provide an optional Kubernetes cluster that can be enabled as described in this +[blog post](https://blog.docker.com/2018/07/kubernetes-is-now-available-in-docker-desktop-stable-channel/). Assuming this is enabled +using this backend will auto-configure itself from the `docker-for-desktop` context that Docker creates in your `~/.kube/config` file. +If your config file is in a different location you should set the `KUBECONFIG` environment variable appropriately. + +### `cloud` and `cloud-url` + +These closely related backends configure the tests to use an arbitrary Kubernetes cluster running in the cloud or otherwise. + +The `cloud` backend auto-configures the cluster to use from your K8S config file, this is assumed to be `~/.kube/config` unless the +`KUBECONFIG` environment variable is set to override this location. By default this will use whatever your current context is in the +config file, to use an alternative context from your config file you can specify the `--context ` flag with the desired context. + +The `cloud-url` backend configures the cluster to simply use a K8S master URL, this should be supplied via the +`--spark-master ` flag. ## Re-using Docker Images @@ -41,12 +71,127 @@ The Spark code to test is handed to the integration test system via a tarball. H * `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. -TODO: Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current tree. +This Tarball should be created by first running `dev/make-distribution.sh` passing the `--tgz` flag and `-Pkubernetes` as one of the +options to ensure that Kubernetes support is included in the distribution. For more details on building a runnable distribution please +see the [Building Spark](https://spark.apache.org/docs/latest/building-spark.html#building-a-runnable-distribution) documentation. + +**TODO:** Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current tree. ## Customizing the Namespace and Service Account -* `--namespace ` - set `` to the namespace in which the tests should be run. -* `--service-account ` - set `` to the name of the Kubernetes service account to -use in the namespace specified by the `--namespace`. The service account is expected to have permissions to get, list, watch, +If no namespace is specified then a temporary namespace will be created and deleted during the test run. Similarly if no service +account is specified then the `default` service account for the namespace will be used. + +Using the `--namespace ` flag sets `` to the namespace in which the tests should be run. If this is supplied +then the tests assume this namespace exists in the K8S cluster and will not attempt to create it. Additionally this namespace must +have an appropriately authorized service account which can be customised via the `--service-account` flag. + +The `--service-account ` flag sets `` to the name of the Kubernetes service account to +use in the namespace specified by the `--namespace` flag. The service account is expected to have permissions to get, list, watch, and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. + +# Running the Test Directly + +If you prefer to run just the integration tests directly then you can customise the behaviour via properties passed to Maven using the +`-Dproperty=value` option e.g. + + mvn integration-test -am -pl :spark-kubernetes-integration-tests_2.11 \ + -Pkubernetes -Phadoop-2.7 -Dhadoop.version=2.7.3 \ + -Dspark.kubernetes.test.sparkTgz=spark-3.0.0-SNAPSHOT-bin-example.tgz \ + -Dspark.kubernetes.test.imageTag=sometag \ + -Dspark.kubernetes.test.imageRepo=docker.io/somerepo \ + -Dspark.kubernetes.test.namespace=spark-int-tests \ + -Dspark.kubernetes.test.deployMode=docker-for-desktop \ + -Dtest.include.tags=k8s + + +## Available Maven Properties + +The following are the available Maven properties that can be passed. For the most part these correspond to flags passed to the +wrapper scripts and using the wrapper scripts will simply set these appropriately behind the scenes. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyDescriptionDefault
spark.kubernetes.test.sparkTgz + A runnable Spark distribution to test. +
spark.kubernetes.test.unpackSparkDir + The directory where the runnable Spark distribution will be unpacked. + ${project.build.directory}/spark-dist-unpacked
spark.kubernetes.test.deployMode + The integration test backend to use. Acceptable values are minikube, docker-for-desktop, + cloud and cloud-url. + minikube
spark.kubernetes.test.kubeConfigContext + When using the cloud backend specifies the context from the users K8S config file that should be used as the + target cluster for integration testing. If not set and using the cloud backend then your current context + will be used. +
spark.kubernetes.test.master + When using the cloud-url backend must be specified to indicate the K8S master URL to communicate with. +
spark.kubernetes.test.imageTag + A specific image tag to use, when set assumes images with those tags are already built and available in the specified image + repository. When set to N/A (the default) fresh images will be built. + N/A +
spark.kubernetes.test.imageTagFile + A file containing the image tag to use, if no specific image tag is set then fresh images will be built with a generated + tag and that tag written to this file. + ${project.build.directory}/imageTag.txt
spark.kubernetes.test.imageRepo + The Docker image repository that contains the images to be used if a specific image tag is set or to which the images will + be pushed to if fresh images are being built. + docker.io/kubespark
spark.kubernetes.test.namespace + A specific Kubernetes namespace to run the tests in. If specified then the tests assume that this namespace already exists. + When not specified a temporary namespace for the tests will be created and deleted as part of the test run. +
spark.kubernetes.test.serviceAccountName + A specific Kubernetes service account to use for running the tests. If not specified then the namespaces default service + account will be used and that must have sufficient permissions or the tests will fail. +
diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index c3c843e001f2..3c7cc9369047 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -26,6 +26,7 @@ IMAGE_TAG="N/A" SPARK_MASTER= NAMESPACE= SERVICE_ACCOUNT= +CONTEXT= INCLUDE_TAGS="k8s" EXCLUDE_TAGS= SCALA_VERSION="$($TEST_ROOT_DIR/build/mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=scala.binary.version | grep -v '\[' )" @@ -61,6 +62,10 @@ while (( "$#" )); do SERVICE_ACCOUNT="$2" shift ;; + --context) + CONTEXT="$2" + shift + ;; --include-tags) INCLUDE_TAGS="k8s,$2" shift @@ -94,6 +99,11 @@ then properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT ) fi +if [ -n $CONTEXT ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.kubeConfigContext=$CONTEXT ) +fi + if [ -n $SPARK_MASTER ]; then properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER ) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index a07fe1feea3e..07288c97bd52 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -33,11 +33,20 @@ 3.2.2 1.0 kubernetes-integration-tests + + + + ${project.build.directory}/spark-dist-unpacked N/A ${project.build.directory}/imageTag.txt minikube docker.io/kubespark + + + + + @@ -135,6 +144,7 @@ ${spark.kubernetes.test.unpackSparkDir} ${spark.kubernetes.test.imageRepo} ${spark.kubernetes.test.deployMode} + ${spark.kubernetes.test.kubeConfigContext} ${spark.kubernetes.test.master} ${spark.kubernetes.test.namespace} ${spark.kubernetes.test.serviceAccountName} diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index ccfb8e767c52..e8fa4e739083 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -71,18 +71,35 @@ if [[ $IMAGE_TAG == "N/A" ]]; then IMAGE_TAG=$(uuidgen); cd $UNPACKED_SPARK_TGZ - if [[ $DEPLOY_MODE == cloud ]] ; - then - $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build - if [[ $IMAGE_REPO == gcr.io* ]] ; - then - gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG - else - $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push - fi - else - # -m option for minikube. - $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build + + case $DEPLOY_MODE in + cloud|cloud-url) + # Build images + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build + + # Push images appropriately + if [[ $IMAGE_REPO == gcr.io* ]] ; + then + gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG + else + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push + fi + ;; + + docker-for-desktop) + # Only need to build as this will place it in our local Docker repo which is all + # we need for Docker for Desktop to work so no need to also push + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build + ;; + + minikube) + # Only need to build and if we do this with the -m option for minikube we will + # build the images directly using the minikube Docker daemon so no need to push + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build + ;; + *) + echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1 + ;; fi cd - fi diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index 8595d0eab112..cd76c3788176 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -17,6 +17,25 @@ package org.apache.spark.deploy.k8s.integrationtest object TestConstants { - val MINIKUBE_TEST_BACKEND = "minikube" - val GCE_TEST_BACKEND = "gce" + val BACKEND_MINIKUBE = "minikube" + val BACKEND_DOCKER_FOR_DESKTOP = "docker-for-desktop" + val BACKEND_CLOUD = "cloud" + val BACKEND_CLOUD_URL = "cloud-url" + + val CONFIG_KEY_DEPLOY_MODE = "spark.kubernetes.test.deployMode" + val CONFIG_KEY_KUBE_CONFIG_CONTEXT = "spark.kubernetes.test.kubeConfigContext" + val CONFIG_KEY_KUBE_MASTER_URL = "spark.kubernetes.test.master" + + + /* + ${project.build.directory}/spark-dist-unpacked + N/A + ${project.build.directory}/imageTag.txt + minikube + docker.io/kubespark + + + + + */ } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala index 663f8b6523ac..ff6fe5b4b0aa 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -27,4 +27,36 @@ object Utils extends Logging { val resource = createResource try f.apply(resource) finally resource.close() } + + def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { + require(rawMasterURL.startsWith("k8s://"), + "Kubernetes master URL must start with k8s://.") + val masterWithoutK8sPrefix = rawMasterURL.substring("k8s://".length) + + // To handle master URLs, e.g., k8s://host:port. + if (!masterWithoutK8sPrefix.contains("://")) { + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + return s"k8s://$resolvedURL" + } + + val masterScheme = new URI(masterWithoutK8sPrefix).getScheme + val resolvedURL = masterScheme.toLowerCase match { + case "https" => + masterWithoutK8sPrefix + case "http" => + logWarning("Kubernetes master URL uses HTTP instead of HTTPS.") + masterWithoutK8sPrefix + case null => + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + resolvedURL + case _ => + throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) + } + + s"k8s://$resolvedURL" + } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index e83a56a4b973..4b1dbb60f5ce 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -18,6 +18,8 @@ package org.apache.spark.deploy.k8s.integrationtest.backend import io.fabric8.kubernetes.client.DefaultKubernetesClient +import org.apache.spark.deploy.k8s.integrationtest.TestConstants._ +import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.{KubeConfigBackend, CloudTestBackend} import org.apache.spark.deploy.k8s.integrationtest.backend.docker.DockerForDesktopBackend import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend @@ -28,18 +30,16 @@ private[spark] trait IntegrationTestBackend { } private[spark] object IntegrationTestBackendFactory { - val deployModeConfigKey = "spark.kubernetes.test.deployMode" - - val backendMinikube = "minikube" - val backendDockerForDesktop = "docker-for-desktop" - def getTestBackend: IntegrationTestBackend = { - val deployMode = Option(System.getProperty(deployModeConfigKey)) - .getOrElse(backendMinikube) + val deployMode = Option(System.getProperty(CONFIG_KEY_DEPLOY_MODE)) + .getOrElse(BACKEND_MINIKUBE) deployMode match { - case `backendMinikube` => MinikubeTestBackend - case `backendDockerForDesktop` => DockerForDesktopBackend - case _ => throw new IllegalArgumentException("Invalid " + deployModeConfigKey + ": " + deployMode) + case BACKEND_MINIKUBE => MinikubeTestBackend + case BACKEND_CLOUD => new KubeConfigBackend(null) + case BACKEND_CLOUD_URL => CloudTestBackend + case BACKEND_DOCKER_FOR_DESKTOP => DockerForDesktopBackend + case _ => throw new IllegalArgumentException("Invalid " + + CONFIG_KEY_DEPLOY_MODE + ": " + deployMode) } } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala new file mode 100644 index 000000000000..bbbedfacaa87 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.cloud + +import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} +import org.apache.spark.deploy.k8s.integrationtest.{TestConstants, Utils} +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object CloudTestBackend extends IntegrationTestBackend { + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + val masterUrl = Option(System.getProperty(TestConstants.CONFIG_KEY_KUBE_MASTER_URL)) + .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) + val k8sConf = new ConfigBuilder() + .withApiVersion("v1") + .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) + .build() + defaultClient = new DefaultKubernetesClient(k8sConf) + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala new file mode 100644 index 000000000000..94701002a1fa --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.cloud + +import java.nio.file.Paths + +import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} +import org.apache.spark.deploy.k8s.integrationtest.TestConstants +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend +import org.apache.spark.internal.Logging + +private[spark] class KubeConfigBackend(var context: String) + extends IntegrationTestBackend with Logging { + // If no context supplied see if one was specified in the system properties supplied + // to the tests + if (context == null) { + context = System.getProperty(TestConstants.CONFIG_KEY_KUBE_CONFIG_CONTEXT) + } + logInfo(s"K8S Integration tests will run against " + + s"${if (context != null) s"context ${context}" else "default context"} " + + s" from users K8S config file") + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + // Auto-configure K8S client from K8S config file + System.setProperty(Config.KUBERNETES_AUTH_TRYKUBECONFIG_SYSTEM_PROPERTY, "true"); + val userHome = System.getProperty("user.home") + System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, + Option(System.getenv("KUBECONFIG")) + .getOrElse(Paths.get(userHome, ".kube", "config").toFile.getAbsolutePath)) + val config = Config.autoConfigure(context) + + defaultClient = new DefaultKubernetesClient(config) + } + + override def cleanUp(): Unit = { + super.cleanUp() + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala index f7644c98fc6d..81a11ae9dcdc 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/docker/DockerForDesktopBackend.scala @@ -1,68 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.spark.deploy.k8s.integrationtest.backend.docker -import java.nio.file.Paths +import org.apache.spark.deploy.k8s.integrationtest.TestConstants +import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.KubeConfigBackend -import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} -import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils -import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend - -private[spark] object DockerForDesktopBackend extends IntegrationTestBackend { - - private val KUBECTL_STARTUP_TIMEOUT_SECONDS = 15 - - private var defaultClient: DefaultKubernetesClient = _ - private var initialContext = "" - - private def getCurrentContext: String = { - val outputs = executeKubectl("config", "current-context") - assert(outputs.size == 1, "Unexpected amount of output from kubectl config current-context") - outputs.head - } - - private def setContext(context: String): Unit = { - val outputs = executeKubectl("config", "use-context", context) - assert(outputs.size == 1, "Unexpected amount of output from kubectl config use-context") - val errors = outputs.filter(_.startsWith("error")) - assert(errors.size == 0, s"Received errors from kubectl: ${errors.head}") - } - - override def initialize(): Unit = { - // Switch context if necessary - // TODO: If we were using Fabric 8 client 3.1.0 then we could - // instead just use the overload of autoConfigure() that takes the - // desired context avoiding the need to interact with kubectl at all - initialContext = getCurrentContext - if (!initialContext.equals("docker-for-desktop")) { - setContext("docker-for-desktop") - } - - // Auto-configure K8S client from K8S config file - System.setProperty(Config.KUBERNETES_AUTH_TRYKUBECONFIG_SYSTEM_PROPERTY, "true"); - val userHome = System.getProperty("user.home") - System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, - Option(System.getenv("KUBECONFIG")) - .getOrElse(Paths.get(userHome, ".kube", "config").toFile.getAbsolutePath)) - val config = Config.autoConfigure() - - defaultClient = new DefaultKubernetesClient(config) - } - - override def cleanUp(): Unit = { - super.cleanUp() - - // Reset users kubectl context appropriately if necessary - if (!initialContext.equals("docker-for-desktop")) { - setContext(initialContext) - } - } - - override def getKubernetesClient: DefaultKubernetesClient = { - defaultClient - } - - private def executeKubectl(args: String*): Seq[String] = { - ProcessUtils.executeProcess( - Array("kubectl") ++ args, KUBECTL_STARTUP_TIMEOUT_SECONDS, true) - } +private[spark] object DockerForDesktopBackend + extends KubeConfigBackend(TestConstants.BACKEND_DOCKER_FOR_DESKTOP) { } From c1a09a032a27a53bb91b6a9e4c285c4f741ebb52 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Wed, 24 Oct 2018 16:09:17 +0100 Subject: [PATCH 3/7] [SPARK-25809][K8S] Add and use constants for config Clarify code by using appropriate constants from TestConstants rather than manually reusing strings --- .../scripts/setup-integration-test-env.sh | 2 +- .../k8s/integrationtest/KubernetesSuite.scala | 3 ++- .../KubernetesTestComponents.scala | 5 +++-- .../k8s/integrationtest/TestConfig.scala | 6 ++++-- .../k8s/integrationtest/TestConstants.scala | 19 ++++++------------- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index e8fa4e739083..29f42ebeba8f 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -100,7 +100,7 @@ then *) echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1 ;; - fi + esac cd - fi diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index c99a907f98d0..27032560fb68 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -33,6 +33,7 @@ import scala.collection.JavaConverters._ import org.apache.spark.SparkFunSuite import org.apache.spark.deploy.k8s.integrationtest.TestConfig._ +import org.apache.spark.deploy.k8s.integrationtest.TestConstants._ import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} import org.apache.spark.internal.Logging @@ -77,7 +78,7 @@ private[spark] class KubernetesSuite extends SparkFunSuite System.clearProperty(key) } - val sparkDirProp = System.getProperty("spark.kubernetes.test.unpackSparkDir") + val sparkDirProp = System.getProperty(CONFIG_KEY_UNPACK_DIR) require(sparkDirProp != null, "Spark home directory must be provided in system properties.") sparkHomeDir = Paths.get(sparkDirProp) require(sparkHomeDir.toFile.isDirectory, diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 5615d6173eeb..c0b435efb8c9 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -25,15 +25,16 @@ import scala.collection.mutable import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually +import org.apache.spark.deploy.k8s.integrationtest.TestConstants._ import org.apache.spark.internal.Logging private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) { - val namespaceOption = Option(System.getProperty("spark.kubernetes.test.namespace")) + val namespaceOption = Option(System.getProperty(CONFIG_KEY_KUBE_NAMESPACE)) val hasUserSpecifiedNamespace = namespaceOption.isDefined val namespace = namespaceOption.getOrElse(UUID.randomUUID().toString.replaceAll("-", "")) val serviceAccountName = - Option(System.getProperty("spark.kubernetes.test.serviceAccountName")) + Option(System.getProperty(CONFIG_KEY_KUBE_SVC_ACCOUNT)) .getOrElse("default") val kubernetesClient = defaultClient.inNamespace(namespace) val clientConfig = kubernetesClient.getConfiguration diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConfig.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConfig.scala index 5a49e0779160..363ec0a6016b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConfig.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConfig.scala @@ -21,9 +21,11 @@ import java.io.File import com.google.common.base.Charsets import com.google.common.io.Files +import org.apache.spark.deploy.k8s.integrationtest.TestConstants._ + object TestConfig { def getTestImageTag: String = { - val imageTagFileProp = System.getProperty("spark.kubernetes.test.imageTagFile") + val imageTagFileProp = System.getProperty(CONFIG_KEY_IMAGE_TAG_FILE) require(imageTagFileProp != null, "Image tag file must be provided in system properties.") val imageTagFile = new File(imageTagFileProp) require(imageTagFile.isFile, s"No file found for image tag at ${imageTagFile.getAbsolutePath}.") @@ -31,7 +33,7 @@ object TestConfig { } def getTestImageRepo: String = { - val imageRepo = System.getProperty("spark.kubernetes.test.imageRepo") + val imageRepo = System.getProperty(CONFIG_KEY_IMAGE_REPO) require(imageRepo != null, "Image repo must be provided in system properties.") imageRepo } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index cd76c3788176..71f5cc2081ec 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -25,17 +25,10 @@ object TestConstants { val CONFIG_KEY_DEPLOY_MODE = "spark.kubernetes.test.deployMode" val CONFIG_KEY_KUBE_CONFIG_CONTEXT = "spark.kubernetes.test.kubeConfigContext" val CONFIG_KEY_KUBE_MASTER_URL = "spark.kubernetes.test.master" - - - /* - ${project.build.directory}/spark-dist-unpacked - N/A - ${project.build.directory}/imageTag.txt - minikube - docker.io/kubespark - - - - - */ + val CONFIG_KEY_KUBE_NAMESPACE = "spark.kubernetes.test.namespace" + val CONFIG_KEY_KUBE_SVC_ACCOUNT = "spark.kubernetes.test.serviceAccountName" + val CONFIG_KEY_IMAGE_TAG = "spark.kubernetes.test.imageTagF" + val CONFIG_KEY_IMAGE_TAG_FILE = "spark.kubernetes.test.imageTagFile" + val CONFIG_KEY_IMAGE_REPO = "spark.kubernetes.test.imageRepo" + val CONFIG_KEY_UNPACK_DIR = "spark.kubernetes.test.unpackSparkDir" } From 8d409373c1c5a35f966e7533a28f2df628e9d60e Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 26 Oct 2018 10:13:50 +0100 Subject: [PATCH 4/7] [SPARK-25809][K8S] Add TODOs for open issue Identified an open issue with spark-submit only ever auto-configuring itself from the current KUBECONFIG context which results in errors if trying to use a non-default context for integration tests. --- .../spark/deploy/k8s/SparkKubernetesClientFactory.scala | 5 +++++ .../integrationtest/backend/cloud/KubeConfigBackend.scala | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala index c47e78cbf19e..f7d73109fba8 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala @@ -42,6 +42,9 @@ private[spark] object SparkKubernetesClientFactory { sparkConf: SparkConf, defaultServiceAccountToken: Option[File], defaultServiceAccountCaCert: Option[File]): KubernetesClient = { + + // TODO Support configurable context + val oauthTokenFileConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_FILE_CONF_SUFFIX" val oauthTokenConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_CONF_SUFFIX" val oauthTokenFile = sparkConf.getOption(oauthTokenFileConf) @@ -63,6 +66,8 @@ private[spark] object SparkKubernetesClientFactory { .getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX") val dispatcher = new Dispatcher( ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher")) + + // TODO Create builder in a way that respects configurable context val config = new ConfigBuilder() .withApiVersion("v1") .withMasterUrl(master) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala index 94701002a1fa..a5d71976d8d4 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala @@ -31,7 +31,7 @@ private[spark] class KubeConfigBackend(var context: String) context = System.getProperty(TestConstants.CONFIG_KEY_KUBE_CONFIG_CONTEXT) } logInfo(s"K8S Integration tests will run against " + - s"${if (context != null) s"context ${context}" else "default context"} " + + s"${if (context != null) s"context ${context}" else "default context"}" + s" from users K8S config file") private var defaultClient: DefaultKubernetesClient = _ From 0246ea1bb8a0c04b2c41edfe9e47e7e8468a9660 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 26 Oct 2018 10:45:23 +0100 Subject: [PATCH 5/7] [SPARK-25809][K8S] Further simplify code - Combine cloud backends into one - Remove unecessary manipulation of System properties - Documentation tweaks - Ensure we clean up Spark Master URL when overriding master URL in KubeConfigBackend - Reuse utils method from Spark Core rather than duplicating in integration tests module - Fix Bash indentation issues - Insert extra new line prior to error output --- .../kubernetes/integration-tests/README.md | 112 ++++++++++-------- .../scripts/setup-integration-test-env.sh | 8 +- .../k8s/integrationtest/ProcessUtils.scala | 2 +- .../k8s/integrationtest/TestConstants.scala | 1 - .../deploy/k8s/integrationtest/Utils.scala | 32 ----- .../backend/IntegrationTestBackend.scala | 5 +- .../backend/cloud/CloudTestBackend.scala | 40 ------- .../backend/cloud/KubeConfigBackend.scala | 32 +++-- 8 files changed, 90 insertions(+), 142 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index de5241693eb5..2e914271f293 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -26,38 +26,40 @@ The main useful options are outlined below. ## Using a different backend -The integration test backend i.e. the K8S cluster used for testing is controlled by the `--deploy-mode` option. By default this -is set to `minikube`, the available backends are their perquisites are as follows. +The integration test backend i.e. the K8S cluster used for testing is controlled by the `--deploy-mode` option. By +default this is set to `minikube`, the available backends are their perequisites are as follows. ### `minikube` -Uses the local `minikube` cluster, this requires that `minikube` 0.23.0 or greater be installed and that it be allocated at least -4 CPUs and 6GB memory (some users have reported success with as few as 3 CPUs and 4GB memory). The tests will check if `minikube` -is started and abort early if it isn't currently running. +Uses the local `minikube` cluster, this requires that `minikube` 0.23.0 or greater be installed and that it be allocated +at least 4 CPUs and 6GB memory (some users have reported success with as few as 3 CPUs and 4GB memory). The tests will +check if `minikube` is started and abort early if it isn't currently running. ### `docker-for-desktop` Since July 2018 Docker for Desktop provide an optional Kubernetes cluster that can be enabled as described in this -[blog post](https://blog.docker.com/2018/07/kubernetes-is-now-available-in-docker-desktop-stable-channel/). Assuming this is enabled -using this backend will auto-configure itself from the `docker-for-desktop` context that Docker creates in your `~/.kube/config` file. -If your config file is in a different location you should set the `KUBECONFIG` environment variable appropriately. +[blog post](https://blog.docker.com/2018/07/kubernetes-is-now-available-in-docker-desktop-stable-channel/). Assuming +this is enabled using this backend will auto-configure itself from the `docker-for-desktop` context that Docker creates +in your `~/.kube/config` file. If your config file is in a different location you should set the `KUBECONFIG` +environment variable appropriately. -### `cloud` and `cloud-url` +### `cloud` -These closely related backends configure the tests to use an arbitrary Kubernetes cluster running in the cloud or otherwise. +These cloud backend configures the tests to use an arbitrary Kubernetes cluster running in the cloud or otherwise. -The `cloud` backend auto-configures the cluster to use from your K8S config file, this is assumed to be `~/.kube/config` unless the -`KUBECONFIG` environment variable is set to override this location. By default this will use whatever your current context is in the -config file, to use an alternative context from your config file you can specify the `--context ` flag with the desired context. +The `cloud` backend auto-configures the cluster to use from your K8S config file, this is assumed to be `~/.kube/config` +unless the `KUBECONFIG` environment variable is set to override this location. By default this will use whatever your +current context is in the config file, to use an alternative context from your config file you can specify the +`--context ` flag with the desired context. -The `cloud-url` backend configures the cluster to simply use a K8S master URL, this should be supplied via the -`--spark-master ` flag. +You can optionally use a different K8S master URL than the one your K8S config file specified, this should be supplied +via the `--spark-master ` flag. ## Re-using Docker Images By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, -and it is written to file at `target/imageTag.txt`. To reuse the images built in a previous run, or to use a Docker image tag -that you have built by other means already, pass the tag to the test script: +and it is written to file at `target/imageTag.txt`. To reuse the images built in a previous run, or to use a Docker +image tag that you have built by other means already, pass the tag to the test script: dev/dev-run-integration-tests.sh --image-tag @@ -67,34 +69,40 @@ where if you still want to use images that were built before by the test framewo ## Spark Distribution Under Test -The Spark code to test is handed to the integration test system via a tarball. Here is the option that is used to specify the tarball: +The Spark code to test is handed to the integration test system via a tarball. Here is the option that is used to +specify the tarball: * `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. -This Tarball should be created by first running `dev/make-distribution.sh` passing the `--tgz` flag and `-Pkubernetes` as one of the -options to ensure that Kubernetes support is included in the distribution. For more details on building a runnable distribution please -see the [Building Spark](https://spark.apache.org/docs/latest/building-spark.html#building-a-runnable-distribution) documentation. +This Tarball should be created by first running `dev/make-distribution.sh` passing the `--tgz` flag and `-Pkubernetes` +as one of the options to ensure that Kubernetes support is included in the distribution. For more details on building a +runnable distribution please see the +[Building Spark](https://spark.apache.org/docs/latest/building-spark.html#building-a-runnable-distribution) +documentation. -**TODO:** Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current tree. +**TODO:** Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current +tree. ## Customizing the Namespace and Service Account -If no namespace is specified then a temporary namespace will be created and deleted during the test run. Similarly if no service -account is specified then the `default` service account for the namespace will be used. +If no namespace is specified then a temporary namespace will be created and deleted during the test run. Similarly if +no service account is specified then the `default` service account for the namespace will be used. -Using the `--namespace ` flag sets `` to the namespace in which the tests should be run. If this is supplied -then the tests assume this namespace exists in the K8S cluster and will not attempt to create it. Additionally this namespace must -have an appropriately authorized service account which can be customised via the `--service-account` flag. +Using the `--namespace ` flag sets `` to the namespace in which the tests should be run. If this +is supplied then the tests assume this namespace exists in the K8S cluster and will not attempt to create it. +Additionally this namespace must have an appropriately authorized service account which can be customised via the +`--service-account` flag. -The `--service-account ` flag sets `` to the name of the Kubernetes service account to -use in the namespace specified by the `--namespace` flag. The service account is expected to have permissions to get, list, watch, -and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account -in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. +The `--service-account ` flag sets `` to the name of the Kubernetes service +account to use in the namespace specified by the `--namespace` flag. The service account is expected to have permissions +to get, list, watch, and create pods. For clusters with RBAC turned on, it's important that the right permissions are +granted to the service account in the namespace through an appropriate role and role binding. A reference RBAC +configuration is provided in `dev/spark-rbac.yaml`. # Running the Test Directly -If you prefer to run just the integration tests directly then you can customise the behaviour via properties passed to Maven using the -`-Dproperty=value` option e.g. +If you prefer to run just the integration tests directly, then you can customise the behaviour via passing system +properties to Maven. For example: mvn integration-test -am -pl :spark-kubernetes-integration-tests_2.11 \ -Pkubernetes -Phadoop-2.7 -Dhadoop.version=2.7.3 \ @@ -108,8 +116,8 @@ If you prefer to run just the integration tests directly then you can customise ## Available Maven Properties -The following are the available Maven properties that can be passed. For the most part these correspond to flags passed to the -wrapper scripts and using the wrapper scripts will simply set these appropriately behind the scenes. +The following are the available Maven properties that can be passed. For the most part these correspond to flags passed +to the wrapper scripts and using the wrapper scripts will simply set these appropriately behind the scenes. @@ -134,63 +142,65 @@ wrapper scripts and using the wrapper scripts will simply set these appropriatel diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh index 29f42ebeba8f..a4a9f5b7da13 100755 --- a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -73,7 +73,7 @@ then cd $UNPACKED_SPARK_TGZ case $DEPLOY_MODE in - cloud|cloud-url) + cloud) # Build images $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build @@ -86,18 +86,18 @@ then fi ;; - docker-for-desktop) + docker-for-desktop) # Only need to build as this will place it in our local Docker repo which is all # we need for Docker for Desktop to work so no need to also push $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build ;; - minikube) + minikube) # Only need to build and if we do this with the -m option for minikube we will # build the images directly using the minikube Docker daemon so no need to push $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build ;; - *) + *) echo "Unrecognized deploy mode $DEPLOY_MODE" && exit 1 ;; esac diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala index 78e0083b4716..004a942c1cdb 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -41,7 +41,7 @@ object ProcessUtils extends Logging { assert(proc.waitFor(timeout, TimeUnit.SECONDS), s"Timed out while executing ${fullCommand.mkString(" ")}") assert(proc.exitValue == 0, - s"Failed to execute ${fullCommand.mkString(" ")}${if (dumpErrors) outputLines.mkString("\n")}") + s"Failed to execute ${fullCommand.mkString(" ")}${if (dumpErrors) "\n" + outputLines.mkString("\n")}") outputLines } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index 71f5cc2081ec..eeae70cd6857 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -20,7 +20,6 @@ object TestConstants { val BACKEND_MINIKUBE = "minikube" val BACKEND_DOCKER_FOR_DESKTOP = "docker-for-desktop" val BACKEND_CLOUD = "cloud" - val BACKEND_CLOUD_URL = "cloud-url" val CONFIG_KEY_DEPLOY_MODE = "spark.kubernetes.test.deployMode" val CONFIG_KEY_KUBE_CONFIG_CONTEXT = "spark.kubernetes.test.kubeConfigContext" diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala index ff6fe5b4b0aa..663f8b6523ac 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -27,36 +27,4 @@ object Utils extends Logging { val resource = createResource try f.apply(resource) finally resource.close() } - - def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { - require(rawMasterURL.startsWith("k8s://"), - "Kubernetes master URL must start with k8s://.") - val masterWithoutK8sPrefix = rawMasterURL.substring("k8s://".length) - - // To handle master URLs, e.g., k8s://host:port. - if (!masterWithoutK8sPrefix.contains("://")) { - val resolvedURL = s"https://$masterWithoutK8sPrefix" - logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + - s"URL is $resolvedURL.") - return s"k8s://$resolvedURL" - } - - val masterScheme = new URI(masterWithoutK8sPrefix).getScheme - val resolvedURL = masterScheme.toLowerCase match { - case "https" => - masterWithoutK8sPrefix - case "http" => - logWarning("Kubernetes master URL uses HTTP instead of HTTPS.") - masterWithoutK8sPrefix - case null => - val resolvedURL = s"https://$masterWithoutK8sPrefix" - logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + - s"URL is $resolvedURL.") - resolvedURL - case _ => - throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) - } - - s"k8s://$resolvedURL" - } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index 4b1dbb60f5ce..7bf324c6c4a1 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -19,7 +19,7 @@ package org.apache.spark.deploy.k8s.integrationtest.backend import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.apache.spark.deploy.k8s.integrationtest.TestConstants._ -import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.{KubeConfigBackend, CloudTestBackend} +import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.KubeConfigBackend import org.apache.spark.deploy.k8s.integrationtest.backend.docker.DockerForDesktopBackend import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend @@ -35,8 +35,7 @@ private[spark] object IntegrationTestBackendFactory { .getOrElse(BACKEND_MINIKUBE) deployMode match { case BACKEND_MINIKUBE => MinikubeTestBackend - case BACKEND_CLOUD => new KubeConfigBackend(null) - case BACKEND_CLOUD_URL => CloudTestBackend + case BACKEND_CLOUD => new KubeConfigBackend(System.getProperty(CONFIG_KEY_KUBE_CONFIG_CONTEXT)) case BACKEND_DOCKER_FOR_DESKTOP => DockerForDesktopBackend case _ => throw new IllegalArgumentException("Invalid " + CONFIG_KEY_DEPLOY_MODE + ": " + deployMode) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala deleted file mode 100644 index bbbedfacaa87..000000000000 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.deploy.k8s.integrationtest.backend.cloud - -import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} -import org.apache.spark.deploy.k8s.integrationtest.{TestConstants, Utils} -import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend - -private[spark] object CloudTestBackend extends IntegrationTestBackend { - - private var defaultClient: DefaultKubernetesClient = _ - - override def initialize(): Unit = { - val masterUrl = Option(System.getProperty(TestConstants.CONFIG_KEY_KUBE_MASTER_URL)) - .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) - val k8sConf = new ConfigBuilder() - .withApiVersion("v1") - .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) - .build() - defaultClient = new DefaultKubernetesClient(k8sConf) - } - - override def getKubernetesClient: DefaultKubernetesClient = { - defaultClient - } -} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala index a5d71976d8d4..333526ba3ef9 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/KubeConfigBackend.scala @@ -18,18 +18,16 @@ package org.apache.spark.deploy.k8s.integrationtest.backend.cloud import java.nio.file.Paths +import io.fabric8.kubernetes.client.utils.Utils import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} +import org.apache.commons.lang3.StringUtils import org.apache.spark.deploy.k8s.integrationtest.TestConstants import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend import org.apache.spark.internal.Logging +import org.apache.spark.util.Utils.checkAndGetK8sMasterUrl private[spark] class KubeConfigBackend(var context: String) extends IntegrationTestBackend with Logging { - // If no context supplied see if one was specified in the system properties supplied - // to the tests - if (context == null) { - context = System.getProperty(TestConstants.CONFIG_KEY_KUBE_CONFIG_CONTEXT) - } logInfo(s"K8S Integration tests will run against " + s"${if (context != null) s"context ${context}" else "default context"}" + s" from users K8S config file") @@ -38,13 +36,27 @@ private[spark] class KubeConfigBackend(var context: String) override def initialize(): Unit = { // Auto-configure K8S client from K8S config file - System.setProperty(Config.KUBERNETES_AUTH_TRYKUBECONFIG_SYSTEM_PROPERTY, "true"); - val userHome = System.getProperty("user.home") - System.setProperty(Config.KUBERNETES_KUBECONFIG_FILE, - Option(System.getenv("KUBECONFIG")) - .getOrElse(Paths.get(userHome, ".kube", "config").toFile.getAbsolutePath)) + if (Utils.getSystemPropertyOrEnvVar(Config.KUBERNETES_KUBECONFIG_FILE, null: String) == null) { + // Fabric 8 client will automatically assume a default location in this case + logWarning(s"No explicit KUBECONFIG specified, will assume .kube/config under your home directory") + } val config = Config.autoConfigure(context) + // If an explicit master URL was specified then override that detected from the + // K8S config if it is different + var masterUrl = Option(System.getProperty(TestConstants.CONFIG_KEY_KUBE_MASTER_URL)) + .getOrElse(null) + if (StringUtils.isNotBlank(masterUrl)) { + // Clean up master URL which would have been specified in Spark format into a normal + // K8S master URL + masterUrl = checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "") + if (!StringUtils.equals(config.getMasterUrl, masterUrl)) { + logInfo(s"Overriding K8S master URL ${config.getMasterUrl} from K8S config file " + + s"with user specified master URL ${masterUrl}") + config.setMasterUrl(masterUrl) + } + } + defaultClient = new DefaultKubernetesClient(config) } From 41af2204db8f85f4ebdad4538a26f24e269a45ee Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Tue, 30 Oct 2018 14:49:05 +0000 Subject: [PATCH 6/7] [SPARK-25809][K8S] Add extra profile to README --- resource-managers/kubernetes/integration-tests/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 2e914271f293..64f8e77597eb 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -8,7 +8,8 @@ title: Spark on Kubernetes Integration Tests Note that the integration test framework is currently being heavily revised and is subject to change. Note that currently the integration tests only run with Java 8. -The simplest way to run the integration tests is to install and run Minikube, then run the following: +The simplest way to run the integration tests is to install and run Minikube, then run the following from this +directory: dev/dev-run-integration-tests.sh @@ -105,7 +106,8 @@ If you prefer to run just the integration tests directly, then you can customise properties to Maven. For example: mvn integration-test -am -pl :spark-kubernetes-integration-tests_2.11 \ - -Pkubernetes -Phadoop-2.7 -Dhadoop.version=2.7.3 \ + -Pkubernetes -Pkubernetes-integration-tests \ + -Phadoop-2.7 -Dhadoop.version=2.7.3 \ -Dspark.kubernetes.test.sparkTgz=spark-3.0.0-SNAPSHOT-bin-example.tgz \ -Dspark.kubernetes.test.imageTag=sometag \ -Dspark.kubernetes.test.imageRepo=docker.io/somerepo \ From cf7ef362352365c7aab4c927db3ea8240fcfaf21 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Wed, 31 Oct 2018 10:07:32 +0000 Subject: [PATCH 7/7] [SPARK-25809][K8S] Reference follow up ticket in comments --- .../spark/deploy/k8s/SparkKubernetesClientFactory.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala index f7d73109fba8..77bd66b608e7 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala @@ -43,7 +43,7 @@ private[spark] object SparkKubernetesClientFactory { defaultServiceAccountToken: Option[File], defaultServiceAccountCaCert: Option[File]): KubernetesClient = { - // TODO Support configurable context + // TODO [SPARK-25887] Support configurable context val oauthTokenFileConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_FILE_CONF_SUFFIX" val oauthTokenConf = s"$kubernetesAuthConfPrefix.$OAUTH_TOKEN_CONF_SUFFIX" @@ -67,7 +67,7 @@ private[spark] object SparkKubernetesClientFactory { val dispatcher = new Dispatcher( ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher")) - // TODO Create builder in a way that respects configurable context + // TODO [SPARK-25887] Create builder in a way that respects configurable context val config = new ConfigBuilder() .withApiVersion("v1") .withMasterUrl(master)
spark.kubernetes.test.deployMode - The integration test backend to use. Acceptable values are minikube, docker-for-desktop, - cloud and cloud-url. + The integration test backend to use. Acceptable values are minikube, + docker-for-desktop and cloud. minikube
spark.kubernetes.test.kubeConfigContext - When using the cloud backend specifies the context from the users K8S config file that should be used as the - target cluster for integration testing. If not set and using the cloud backend then your current context - will be used. + When using the cloud backend specifies the context from the users K8S config file that should be used + as the target cluster for integration testing. If not set and using the cloud backend then your + current context will be used.
spark.kubernetes.test.master - When using the cloud-url backend must be specified to indicate the K8S master URL to communicate with. + When using the cloud-url backend must be specified to indicate the K8S master URL to communicate + with.
spark.kubernetes.test.imageTag - A specific image tag to use, when set assumes images with those tags are already built and available in the specified image - repository. When set to N/A (the default) fresh images will be built. + A specific image tag to use, when set assumes images with those tags are already built and available in the + specified image repository. When set to N/A (the default) fresh images will be built. N/A
spark.kubernetes.test.imageTagFile - A file containing the image tag to use, if no specific image tag is set then fresh images will be built with a generated - tag and that tag written to this file. + A file containing the image tag to use, if no specific image tag is set then fresh images will be built with a + generated tag and that tag written to this file. ${project.build.directory}/imageTag.txt
spark.kubernetes.test.imageRepo - The Docker image repository that contains the images to be used if a specific image tag is set or to which the images will - be pushed to if fresh images are being built. + The Docker image repository that contains the images to be used if a specific image tag is set or to which the + images will be pushed to if fresh images are being built. docker.io/kubespark
spark.kubernetes.test.namespace - A specific Kubernetes namespace to run the tests in. If specified then the tests assume that this namespace already exists. - When not specified a temporary namespace for the tests will be created and deleted as part of the test run. + A specific Kubernetes namespace to run the tests in. If specified then the tests assume that this namespace + already exists. When not specified a temporary namespace for the tests will be created and deleted as part of the + test run.
spark.kubernetes.test.serviceAccountName - A specific Kubernetes service account to use for running the tests. If not specified then the namespaces default service - account will be used and that must have sufficient permissions or the tests will fail. + A specific Kubernetes service account to use for running the tests. If not specified then the namespaces default + service account will be used and that must have sufficient permissions or the tests will fail.