From de62f07bcc33f4d0e995b468c49a8301a99d60d0 Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Wed, 24 Oct 2018 17:24:16 -0700 Subject: [PATCH 1/6] bump to 4.0.0 for kubernetes-client version --- resource-managers/kubernetes/core/pom.xml | 2 +- .../org/apache/spark/deploy/k8s/KubernetesUtils.scala | 9 ++++++--- .../deploy/k8s/features/MountVolumesFeatureStep.scala | 4 +++- .../deploy/k8s/submit/LoggingPodStatusWatcher.scala | 2 +- .../cluster/k8s/ExecutorLifecycleTestUtils.scala | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index 90bac19cba019..e896c32d67020 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -29,7 +29,7 @@ Spark Project Kubernetes kubernetes - 3.0.0 + 4.0.0 diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 8f36fa12aed17..5c36ba7014bc2 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s import scala.collection.JavaConverters._ -import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time} +import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod} import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.util.Utils @@ -157,7 +157,10 @@ private[spark] object KubernetesUtils { }.getOrElse(Seq(("container state", "N/A"))) } - def formatTime(time: Time): String = { - if (time != null) time.getTime else "N/A" + def formatTime(time: String): String = { + time match { + case null => "N/A" + case _ => time + } } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala index bb0e2b3128efd..a16e901ef7c82 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala @@ -56,8 +56,10 @@ private[spark] class MountVolumesFeatureStep( val volumeBuilder = spec.volumeConf match { case KubernetesHostPathVolumeConf(hostPath) => + // TODO: Backwards compatibility allows for empty string, but specify + // hostPath type here new VolumeBuilder() - .withHostPath(new HostPathVolumeSource(hostPath)) + .withHostPath(new HostPathVolumeSource(hostPath, "")) case KubernetesPVCVolumeConf(claimName) => new VolumeBuilder() diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala index 1889fe5eb3e9b..79b55bc37afcd 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala @@ -20,7 +20,7 @@ import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.collection.JavaConverters._ -import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time} +import io.fabric8.kubernetes.api.model.Pod import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher} import io.fabric8.kubernetes.client.Watcher.Action diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala index c6b667ed85e8c..2e883623a4b1c 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorLifecycleTestUtils.scala @@ -82,7 +82,7 @@ object ExecutorLifecycleTestUtils { def deletedExecutor(executorId: Long): Pod = { new PodBuilder(podWithAttachedContainerForId(executorId)) .editOrNewMetadata() - .withNewDeletionTimestamp("523012521") + .withDeletionTimestamp("523012521") .endMetadata() .build() } diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 23453c8957b28..8bf137692221f 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -29,7 +29,7 @@ 1.3.0 1.4.0 - 3.0.0 + 4.0.0 3.2.2 1.0 kubernetes-integration-tests From 3d7a0b994dc6df6c8f20572a00303e4c0a59dc0a Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Wed, 24 Oct 2018 17:46:57 -0700 Subject: [PATCH 2/6] bump to 4.1.0 --- resource-managers/kubernetes/core/pom.xml | 2 +- resource-managers/kubernetes/integration-tests/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index e896c32d67020..b89ea383bf872 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -29,7 +29,7 @@ Spark Project Kubernetes kubernetes - 4.0.0 + 4.1.0 diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 8bf137692221f..a07fe1feea3eb 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -29,7 +29,7 @@ 1.3.0 1.4.0 - 4.0.0 + 4.1.0 3.2.2 1.0 kubernetes-integration-tests From a28ebcf493343b45818c93524b879a7044c98022 Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Thu, 25 Oct 2018 10:59:51 -0700 Subject: [PATCH 3/6] comments --- docs/running-on-kubernetes.md | 3 ++- .../scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala | 6 ++---- .../spark/deploy/k8s/features/MountVolumesFeatureStep.scala | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index d629ed3b503ab..88451dc417526 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -45,7 +45,8 @@ logs and remains in "completed" state in the Kubernetes API until it's eventuall Note that in the completed state, the driver pod does *not* use any computational or memory resources. -The driver and executor pod scheduling is handled by Kubernetes. It is possible to schedule the +The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8, and we are +currently running kubernetes-client version 4.1.0. Make sure that any infrastructure additions are weary of said version. It is possible to schedule the driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) using the configuration property for it. It will be possible to use more advanced scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 5c36ba7014bc2..cfa6e3af40f7b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -158,9 +158,7 @@ private[spark] object KubernetesUtils { } def formatTime(time: String): String = { - time match { - case null => "N/A" - case _ => time - } + // TODO: Investigate form `time` comes in + if (time != null) time else "N/A" } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala index a16e901ef7c82..e60259c4a9b5a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/MountVolumesFeatureStep.scala @@ -56,8 +56,7 @@ private[spark] class MountVolumesFeatureStep( val volumeBuilder = spec.volumeConf match { case KubernetesHostPathVolumeConf(hostPath) => - // TODO: Backwards compatibility allows for empty string, but specify - // hostPath type here + /* "" means that no checks will be performed before mounting the hostPath volume */ new VolumeBuilder() .withHostPath(new HostPathVolumeSource(hostPath, "")) From c6b67230f3294cd4d8a6eeb0d8633c98c0808afc Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Thu, 25 Oct 2018 11:15:31 -0700 Subject: [PATCH 4/6] replaced manifests --- dev/deps/spark-deps-hadoop-2.7 | 6 +++--- dev/deps/spark-deps-hadoop-3.1 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 06173f73cc32a..a0c9ea1873381 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -132,13 +132,13 @@ jta-1.1.jar jtransforms-2.4.0.jar jul-to-slf4j-1.7.16.jar kryo-shaded-4.0.2.jar -kubernetes-client-3.0.0.jar -kubernetes-model-2.0.0.jar +kubernetes-client-4.1.0.jar +kubernetes-model-4.1.0.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.9.3.jar log4j-1.2.17.jar -logging-interceptor-3.8.1.jar +logging-interceptor-3.9.1.jar lz4-java-1.5.0.jar machinist_2.11-0.6.1.jar macro-compat_2.11-1.1.1.jar diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 62fddf084c8bc..fb26e757bf635 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -147,13 +147,13 @@ kerby-pkix-1.0.1.jar kerby-util-1.0.1.jar kerby-xdr-1.0.1.jar kryo-shaded-4.0.2.jar -kubernetes-client-3.0.0.jar -kubernetes-model-2.0.0.jar +kubernetes-client-4.1.0.jar +kubernetes-model-4.1.0.jar leveldbjni-all-1.8.jar libfb303-0.9.3.jar libthrift-0.9.3.jar log4j-1.2.17.jar -logging-interceptor-3.8.1.jar +logging-interceptor-3.9.1.jar lz4-java-1.5.0.jar machinist_2.11-0.6.1.jar macro-compat_2.11-1.1.1.jar From 728d70af1d0917745879362abef2209a760d4f22 Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Thu, 25 Oct 2018 15:11:26 -0700 Subject: [PATCH 5/6] Resolve comments --- docs/running-on-kubernetes.md | 2 +- .../scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index 88451dc417526..6e1c5bda884ff 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -46,7 +46,7 @@ logs and remains in "completed" state in the Kubernetes API until it's eventuall Note that in the completed state, the driver pod does *not* use any computational or memory resources. The driver and executor pod scheduling is handled by Kubernetes. Communication to the Kubernetes API is done via fabric8, and we are -currently running kubernetes-client version 4.1.0. Make sure that any infrastructure additions are weary of said version. It is possible to schedule the +currently running kubernetes-client version 4.1.0. Make sure that when you are making infrastructure additions that you are aware of said version. It is possible to schedule the driver and executor pods on a subset of available nodes through a [node selector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) using the configuration property for it. It will be possible to use more advanced scheduling hints like [node/pod affinities](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) in a future release. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index cfa6e3af40f7b..44c790a1d1b06 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -158,7 +158,8 @@ private[spark] object KubernetesUtils { } def formatTime(time: String): String = { - // TODO: Investigate form `time` comes in + // TODO: Investigate form `time` comes in as there was a DataType + // change from Time to String if (time != null) time else "N/A" } } From b0ba9512020025572ae06de098e0bce62387b60d Mon Sep 17 00:00:00 2001 From: Ilan Filonenko Date: Fri, 26 Oct 2018 10:22:24 -0700 Subject: [PATCH 6/6] took out comment --- .../scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 44c790a1d1b06..0f740454fafc4 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -158,8 +158,6 @@ private[spark] object KubernetesUtils { } def formatTime(time: String): String = { - // TODO: Investigate form `time` comes in as there was a DataType - // change from Time to String if (time != null) time else "N/A" } }