From d4d5877a00567213126c39a84f6dc06da0ef18b2 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 17 Nov 2021 12:48:24 -0800 Subject: [PATCH 1/7] Drop remove reason stats check to make the test less flaky --- .../spark/deploy/k8s/integrationtest/DecommissionSuite.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala index 125012669195..27948c810701 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala @@ -134,9 +134,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite => mainClass = "", expectedDriverLogOnCompletion = Seq( "Finished waiting, stopping Spark", - "Decommission executors", - "Remove reason statistics: (gracefully decommissioned: 1, decommision unfinished: 0, " + - "driver killed: 0, unexpectedly exited: 0)."), + "Decommission executors"), appArgs = Array.empty[String], driverPodChecker = doBasicDriverPyPodCheck, executorPodChecker = checkFirstExecutorPodGetsLabeled, From a73b27c867455c323cc31bdc33b4154d75b8fd0f Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 17 Nov 2021 12:53:52 -0800 Subject: [PATCH 2/7] Drop the RTestSuite it hasn't passed for weeks --- .../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index e608b3aa76ae..eca63a72c7ea 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -44,7 +44,7 @@ import org.apache.spark.internal.config._ class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter with BasicTestsSuite with SparkConfPropagateSuite with SecretsTestsSuite with PythonTestsSuite with ClientModeTestsSuite with PodTemplateSuite - with PVTestsSuite with DepsTestsSuite with DecommissionSuite with RTestsSuite with Logging + with PVTestsSuite with DepsTestsSuite with DecommissionSuite with Logging with Eventually with Matchers { From 12a4096d740e2cfb6842f08ceb5088ecc3a72db4 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 18 Nov 2021 09:51:09 -0800 Subject: [PATCH 3/7] Try and avoid the PV/PVC delete/create race condition during PVTestSuite by iterating on the name --- .../deploy/k8s/integrationtest/PVTestsSuite.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 0362b348e139..34c7907707c7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -28,6 +28,8 @@ import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => import PVTestsSuite._ + var i = 0 + private def setupLocalStorageClass(): Unit = { val scBuilder = new StorageClassBuilder() .withKind("StorageClass") @@ -53,11 +55,13 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => setupLocalStorageClass() + i = i + 1 + val pvBuilder = new PersistentVolumeBuilder() .withKind("PersistentVolume") .withApiVersion("v1") .withNewMetadata() - .withName("test-local-pv") + .withName(f"{PV_NAME}-{i}") .endMetadata() .withNewSpec() .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava) @@ -81,7 +85,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolumeClaim") .withApiVersion("v1") .withNewMetadata() - .withName(PVC_NAME) + .withName(f"{PVC_NAME}-{i}") .endMetadata() .withNewSpec() .withAccessModes("ReadWriteOnce") @@ -105,13 +109,13 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => kubernetesTestComponents .kubernetesClient .persistentVolumeClaims() - .withName(PVC_NAME) + .withName(f"{PVC_NAME}-{i}") .delete() kubernetesTestComponents .kubernetesClient .persistentVolumes() - .withName(PV_NAME) + .withName(f"{PV_NAME}-{i}") .delete() kubernetesTestComponents From e1e61f24dc88993733e7b2c274fa6e4c20fffb26 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Thu, 18 Nov 2021 11:22:17 -0800 Subject: [PATCH 4/7] Fix string sub --- .../spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 34c7907707c7..28f2929f0fd7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -61,7 +61,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolume") .withApiVersion("v1") .withNewMetadata() - .withName(f"{PV_NAME}-{i}") + .withName(s"${PV_NAME}-${i}") .endMetadata() .withNewSpec() .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava) @@ -85,7 +85,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolumeClaim") .withApiVersion("v1") .withNewMetadata() - .withName(f"{PVC_NAME}-{i}") + .withName(s"${PVC_NAME}-${i}") .endMetadata() .withNewSpec() .withAccessModes("ReadWriteOnce") @@ -109,13 +109,13 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => kubernetesTestComponents .kubernetesClient .persistentVolumeClaims() - .withName(f"{PVC_NAME}-{i}") + .withName(s"${PVC_NAME}-${i}") .delete() kubernetesTestComponents .kubernetesClient .persistentVolumes() - .withName(f"{PV_NAME}-{i}") + .withName(s"${PV_NAME}-${i}") .delete() kubernetesTestComponents From b3b1065e5d661c84bce7bcab72fe6bc85cdb73a7 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 22 Nov 2021 10:52:37 -0800 Subject: [PATCH 5/7] Revert "Fix string sub" This reverts commit e1e61f24dc88993733e7b2c274fa6e4c20fffb26. --- .../spark/deploy/k8s/integrationtest/PVTestsSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 28f2929f0fd7..34c7907707c7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -61,7 +61,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolume") .withApiVersion("v1") .withNewMetadata() - .withName(s"${PV_NAME}-${i}") + .withName(f"{PV_NAME}-{i}") .endMetadata() .withNewSpec() .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava) @@ -85,7 +85,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolumeClaim") .withApiVersion("v1") .withNewMetadata() - .withName(s"${PVC_NAME}-${i}") + .withName(f"{PVC_NAME}-{i}") .endMetadata() .withNewSpec() .withAccessModes("ReadWriteOnce") @@ -109,13 +109,13 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => kubernetesTestComponents .kubernetesClient .persistentVolumeClaims() - .withName(s"${PVC_NAME}-${i}") + .withName(f"{PVC_NAME}-{i}") .delete() kubernetesTestComponents .kubernetesClient .persistentVolumes() - .withName(s"${PV_NAME}-${i}") + .withName(f"{PV_NAME}-{i}") .delete() kubernetesTestComponents From c866acfab22ae025a288667a2a193c2d7f4e0683 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 22 Nov 2021 10:52:43 -0800 Subject: [PATCH 6/7] Revert "Try and avoid the PV/PVC delete/create race condition during PVTestSuite by iterating on the name" This reverts commit 12a4096d740e2cfb6842f08ceb5088ecc3a72db4. --- .../deploy/k8s/integrationtest/PVTestsSuite.scala | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 34c7907707c7..0362b348e139 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -28,8 +28,6 @@ import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite._ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => import PVTestsSuite._ - var i = 0 - private def setupLocalStorageClass(): Unit = { val scBuilder = new StorageClassBuilder() .withKind("StorageClass") @@ -55,13 +53,11 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => setupLocalStorageClass() - i = i + 1 - val pvBuilder = new PersistentVolumeBuilder() .withKind("PersistentVolume") .withApiVersion("v1") .withNewMetadata() - .withName(f"{PV_NAME}-{i}") + .withName("test-local-pv") .endMetadata() .withNewSpec() .withCapacity(Map("storage" -> new Quantity("1Gi")).asJava) @@ -85,7 +81,7 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .withKind("PersistentVolumeClaim") .withApiVersion("v1") .withNewMetadata() - .withName(f"{PVC_NAME}-{i}") + .withName(PVC_NAME) .endMetadata() .withNewSpec() .withAccessModes("ReadWriteOnce") @@ -109,13 +105,13 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => kubernetesTestComponents .kubernetesClient .persistentVolumeClaims() - .withName(f"{PVC_NAME}-{i}") + .withName(PVC_NAME) .delete() kubernetesTestComponents .kubernetesClient .persistentVolumes() - .withName(f"{PV_NAME}-{i}") + .withName(PV_NAME) .delete() kubernetesTestComponents From 14022c0b2e3cdb85ac11511e11a5827a920b8070 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 22 Nov 2021 11:31:22 -0800 Subject: [PATCH 7/7] Check for null --- .../k8s/integrationtest/PVTestsSuite.scala | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala index 0362b348e139..4ebe2860af83 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/PVTestsSuite.scala @@ -120,6 +120,28 @@ private[spark] trait PVTestsSuite { k8sSuite: KubernetesSuite => .storageClasses() .withName(STORAGE_NAME) .delete() + + // Wait for them to be deleted + Eventually.eventually(TIMEOUT, INTERVAL) { + assert(kubernetesTestComponents + .kubernetesClient + .persistentVolumeClaims() + .withName(PVC_NAME) + .get() === null) + + assert(kubernetesTestComponents + .kubernetesClient + .persistentVolumes() + .withName(PV_NAME) + .get() === null) + + assert(kubernetesTestComponents + .kubernetesClient + .storage() + .storageClasses() + .withName(STORAGE_NAME) + .get() === null) + } } private def checkPVs(pod: Pod, file: String) = {