From 20f1aff95b9cab7381edc1c107f52725d506a20c Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 18 Apr 2023 12:21:57 +0800 Subject: [PATCH 1/5] [SPARK-43171][K8S] Support dynamic changing unix user in Pod --- .../kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh index 42f4df88f3da9..11160ff024980 100755 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh @@ -30,7 +30,7 @@ set -e # If there is no passwd entry for the container UID, attempt to create one if [ -z "$uidentry" ] ; then if [ -w /etc/passwd ] ; then - echo "$myuid:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd + echo "${SPARK_USER_NAME:-$myuid}:x:$myuid:$mygid:${SPARK_USER_NAME:-anonymous uid}:$SPARK_HOME:/bin/false" >> /etc/passwd else echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" fi From bec5f3f2ec4d3e27fc430aec1c4111a6fea00919 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 28 Apr 2023 16:34:43 +0800 Subject: [PATCH 2/5] IT --- .../k8s/integrationtest/DepsTestsSuite.scala | 26 +++++++++++++++++-- .../k8s/integrationtest/KubernetesSuite.scala | 1 + 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala index aed9c1a854d20..7cc9fcfdbf6af 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala @@ -32,7 +32,7 @@ import org.scalatest.time.{Minutes, Span} import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH} -import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, SPARK_PI_MAIN_CLASS, TIMEOUT} +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{usernameTestTag, INTERVAL, MinikubeTag, SPARK_PI_MAIN_CLASS, TIMEOUT} import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} @@ -269,15 +269,36 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => "Python runtime version check is: True", "Python environment version check is: True", "Python runtime version check for executor is: True"), - Some(outDepsFile)) + depsFile = Some(outDepsFile)) } finally { Files.delete(new File(outDepsFile).toPath) } } + test("SPARK-43171: Support custom Unix username in Pod", + k8sTestTag, usernameTestTag, MinikubeTag) { + val fileName = Utils.createTempFile( + """#!/usr/bin/env bash + |export IS_CUSTOM_PYTHON=1 + |echo "login user is `id -un`" + |python3 "$@" + """.stripMargin, HOST_PATH) + Utils.createTarGzFile(s"$HOST_PATH/$fileName", s"$HOST_PATH/$fileName.tgz") + sparkAppConf + .set(ARCHIVES.key, s"$HOST_PATH/$fileName.tgz#test_env") + .set(PYSPARK_PYTHON.key, s"./test_env/$fileName") + .set("spark.kubernetes.driverEnv.SPARK_USER_NAME", "spark-kube") + .set("spark.executorEnv.SPARK_USER_NAME", "spark-kube") + val pySparkFiles = Utils.getTestFileAbsolutePath("python_executable_check.py", sparkHomeDir) + testPython(pySparkFiles, + expectedDriverLogs = Seq("login user is spark-kube"), + expectedExecutorLogs = Seq("login user is spark-kube")) + } + private def testPython( pySparkFiles: String, expectedDriverLogs: Seq[String], + expectedExecutorLogs: Seq[String] = Seq.empty, depsFile: Option[String] = None, env: Map[String, String] = Map.empty[String, String]): Unit = { tryDepsTest { @@ -286,6 +307,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => appResource = pySparkFiles, mainClass = "", expectedDriverLogOnCompletion = expectedDriverLogs, + expectedExecutorLogOnCompletion = expectedExecutorLogs, appArgs = Array("python3"), driverPodChecker = doBasicDriverPyPodCheck, executorPodChecker = doBasicExecutorPyPodCheck, diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 78839ee610336..1af7ae252704a 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -618,6 +618,7 @@ class KubernetesSuite extends SparkFunSuite private[spark] object KubernetesSuite { val k8sTestTag = Tag("k8s") + val usernameTestTag = Tag("username") val localTestTag = Tag("local") val schedulingTestTag = Tag("schedule") val decomTestTag = Tag("decom") From 049a9840d73c75fa2f66457dd3a755ed5d4f1551 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 28 Apr 2023 17:22:10 +0800 Subject: [PATCH 3/5] Revert "IT" This reverts commit bec5f3f2ec4d3e27fc430aec1c4111a6fea00919. --- .../k8s/integrationtest/DepsTestsSuite.scala | 26 ++----------------- .../k8s/integrationtest/KubernetesSuite.scala | 1 - 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala index 7cc9fcfdbf6af..aed9c1a854d20 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala @@ -32,7 +32,7 @@ import org.scalatest.time.{Minutes, Span} import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH} -import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{usernameTestTag, INTERVAL, MinikubeTag, SPARK_PI_MAIN_CLASS, TIMEOUT} +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, SPARK_PI_MAIN_CLASS, TIMEOUT} import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} @@ -269,36 +269,15 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => "Python runtime version check is: True", "Python environment version check is: True", "Python runtime version check for executor is: True"), - depsFile = Some(outDepsFile)) + Some(outDepsFile)) } finally { Files.delete(new File(outDepsFile).toPath) } } - test("SPARK-43171: Support custom Unix username in Pod", - k8sTestTag, usernameTestTag, MinikubeTag) { - val fileName = Utils.createTempFile( - """#!/usr/bin/env bash - |export IS_CUSTOM_PYTHON=1 - |echo "login user is `id -un`" - |python3 "$@" - """.stripMargin, HOST_PATH) - Utils.createTarGzFile(s"$HOST_PATH/$fileName", s"$HOST_PATH/$fileName.tgz") - sparkAppConf - .set(ARCHIVES.key, s"$HOST_PATH/$fileName.tgz#test_env") - .set(PYSPARK_PYTHON.key, s"./test_env/$fileName") - .set("spark.kubernetes.driverEnv.SPARK_USER_NAME", "spark-kube") - .set("spark.executorEnv.SPARK_USER_NAME", "spark-kube") - val pySparkFiles = Utils.getTestFileAbsolutePath("python_executable_check.py", sparkHomeDir) - testPython(pySparkFiles, - expectedDriverLogs = Seq("login user is spark-kube"), - expectedExecutorLogs = Seq("login user is spark-kube")) - } - private def testPython( pySparkFiles: String, expectedDriverLogs: Seq[String], - expectedExecutorLogs: Seq[String] = Seq.empty, depsFile: Option[String] = None, env: Map[String, String] = Map.empty[String, String]): Unit = { tryDepsTest { @@ -307,7 +286,6 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => appResource = pySparkFiles, mainClass = "", expectedDriverLogOnCompletion = expectedDriverLogs, - expectedExecutorLogOnCompletion = expectedExecutorLogs, appArgs = Array("python3"), driverPodChecker = doBasicDriverPyPodCheck, executorPodChecker = doBasicExecutorPyPodCheck, diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 1af7ae252704a..78839ee610336 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -618,7 +618,6 @@ class KubernetesSuite extends SparkFunSuite private[spark] object KubernetesSuite { val k8sTestTag = Tag("k8s") - val usernameTestTag = Tag("username") val localTestTag = Tag("local") val schedulingTestTag = Tag("schedule") val decomTestTag = Tag("decom") From 6fecd5ce286c9da8e204b042b2dc4b5468a88706 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 28 Apr 2023 17:28:17 +0800 Subject: [PATCH 4/5] IT 2 --- .../src/main/dockerfiles/spark/entrypoint.sh | 2 ++ .../k8s/integrationtest/BasicTestsSuite.scala | 19 ++++++++++++++++++- .../k8s/integrationtest/KubernetesSuite.scala | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh index 11160ff024980..3bed78292cbdd 100755 --- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh +++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh @@ -36,6 +36,8 @@ if [ -z "$uidentry" ] ; then fi fi +echo "login user is `id -un`" + if [ -z "$JAVA_HOME" ]; then JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}') fi diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index af74032694db5..bf315f9582982 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -24,7 +24,7 @@ import org.scalatest.matchers.should.Matchers._ import org.scalatest.time.{Seconds, Span} import org.apache.spark.{SparkFunSuite, TestUtils} -import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.SPARK_PI_MAIN_CLASS +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{usernameTestTag, MinikubeTag, SPARK_PI_MAIN_CLASS} import org.apache.spark.launcher.SparkLauncher private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => @@ -168,6 +168,23 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => .exists(envVar => envVar.getName == "SPARK_DRIVER_POD_IP") }) } + + test("SPARK-43171: Support custom Unix username in Pod", + k8sTestTag, usernameTestTag, MinikubeTag) { + sparkAppConf + .set("spark.kubernetes.driverEnv.SPARK_USER_NAME", "spark-kube") + .set("spark.executorEnv.SPARK_USER_NAME", "spark-kube") + runSparkApplicationAndVerifyCompletion( + containerLocalSparkDistroExamplesJar, + SPARK_PI_MAIN_CLASS, + Seq("login user is spark-kube", "Pi is roughly 3"), + Seq("login user is spark-kube"), + appArgs = Array.empty, + doBasicDriverPodCheck, + doBasicExecutorPodCheck, + isJVM = true + ) + } } private[spark] object BasicTestsSuite extends SparkFunSuite { diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 78839ee610336..4b8ba23edd980 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -623,6 +623,7 @@ private[spark] object KubernetesSuite { val decomTestTag = Tag("decom") val rTestTag = Tag("r") val MinikubeTag = Tag("minikube") + val usernameTestTag = Tag("username") val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_DFS_READ_WRITE_TEST = "org.apache.spark.examples.DFSReadWriteTest" val SPARK_MINI_READ_WRITE_TEST = "org.apache.spark.examples.MiniReadWriteTest" From 3ee42bb567f29cc8051144a9a0cac0fc84bcf721 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Fri, 28 Apr 2023 17:55:11 +0800 Subject: [PATCH 5/5] nit --- .../spark/deploy/k8s/integrationtest/BasicTestsSuite.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala index bf315f9582982..f7de43a53652b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala @@ -24,7 +24,7 @@ import org.scalatest.matchers.should.Matchers._ import org.scalatest.time.{Seconds, Span} import org.apache.spark.{SparkFunSuite, TestUtils} -import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{usernameTestTag, MinikubeTag, SPARK_PI_MAIN_CLASS} +import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{usernameTestTag, SPARK_PI_MAIN_CLASS} import org.apache.spark.launcher.SparkLauncher private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => @@ -169,8 +169,7 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite => }) } - test("SPARK-43171: Support custom Unix username in Pod", - k8sTestTag, usernameTestTag, MinikubeTag) { + test("SPARK-43171: Support custom Unix username in Pod", k8sTestTag, usernameTestTag) { sparkAppConf .set("spark.kubernetes.driverEnv.SPARK_USER_NAME", "spark-kube") .set("spark.executorEnv.SPARK_USER_NAME", "spark-kube")