From 48c4b0a6f31b3d05151219f55f8ac545a8236102 Mon Sep 17 00:00:00 2001 From: Yinan Li Date: Wed, 13 Dec 2017 13:05:41 -0800 Subject: [PATCH 1/3] [SPARK-22778][Kubernetes] Added the missing service metadata for KubernetesClusterManager --- core/src/main/scala/org/apache/spark/util/Utils.scala | 6 +++--- .../src/test/scala/org/apache/spark/util/UtilsSuite.scala | 8 ++++---- .../org.apache.spark.scheduler.ExternalClusterManager | 1 + .../deploy/k8s/submit/KubernetesClientApplication.scala | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index fe5b4ea24440..8871870eb868 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2757,7 +2757,7 @@ private[spark] object Utils extends Logging { /** * Check the validity of the given Kubernetes master URL and return the resolved URL. Prefix - * "k8s:" is appended to the resolved URL as the prefix is used by KubernetesClusterManager + * "k8s://" is appended to the resolved URL as the prefix is used by KubernetesClusterManager * in canCreate to determine if the KubernetesClusterManager should be used. */ def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { @@ -2770,7 +2770,7 @@ private[spark] object Utils extends Logging { val resolvedURL = s"https://$masterWithoutK8sPrefix" logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + s"URL is $resolvedURL.") - return s"k8s:$resolvedURL" + return s"k8s://$resolvedURL" } val masterScheme = new URI(masterWithoutK8sPrefix).getScheme @@ -2789,7 +2789,7 @@ private[spark] object Utils extends Logging { throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) } - return s"k8s:$resolvedURL" + s"k8s://$resolvedURL" } } diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 5c4e4ca0cded..eaea6b030c15 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -1148,16 +1148,16 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { test("check Kubernetes master URL") { val k8sMasterURLHttps = Utils.checkAndGetK8sMasterUrl("k8s://https://host:port") - assert(k8sMasterURLHttps === "k8s:https://host:port") + assert(k8sMasterURLHttps === "k8s://https://host:port") val k8sMasterURLHttp = Utils.checkAndGetK8sMasterUrl("k8s://http://host:port") - assert(k8sMasterURLHttp === "k8s:http://host:port") + assert(k8sMasterURLHttp === "k8s://http://host:port") val k8sMasterURLWithoutScheme = Utils.checkAndGetK8sMasterUrl("k8s://127.0.0.1:8443") - assert(k8sMasterURLWithoutScheme === "k8s:https://127.0.0.1:8443") + assert(k8sMasterURLWithoutScheme === "k8s://https://127.0.0.1:8443") val k8sMasterURLWithoutScheme2 = Utils.checkAndGetK8sMasterUrl("k8s://127.0.0.1") - assert(k8sMasterURLWithoutScheme2 === "k8s:https://127.0.0.1") + assert(k8sMasterURLWithoutScheme2 === "k8s://https://127.0.0.1") intercept[IllegalArgumentException] { Utils.checkAndGetK8sMasterUrl("k8s:https://host:port") diff --git a/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager b/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager new file mode 100644 index 000000000000..7965fb133611 --- /dev/null +++ b/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager @@ -0,0 +1 @@ +org.apache.spark.scheduler.cluster.k8s.KubernetesClusterManager \ No newline at end of file diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala index 4d17608c602d..240a1144577b 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientApplication.scala @@ -203,8 +203,8 @@ private[spark] class KubernetesClientApplication extends SparkApplication { val waitForAppCompletion = sparkConf.get(WAIT_FOR_APP_COMPLETION) val appName = sparkConf.getOption("spark.app.name").getOrElse("spark") // The master URL has been checked for validity already in SparkSubmit. - // We just need to get rid of the "k8s:" prefix here. - val master = sparkConf.get("spark.master").substring("k8s:".length) + // We just need to get rid of the "k8s://" prefix here. + val master = sparkConf.get("spark.master").substring("k8s://".length) val loggingInterval = if (waitForAppCompletion) Some(sparkConf.get(REPORT_INTERVAL)) else None val loggingPodStatusWatcher = new LoggingPodStatusWatcherImpl( From c91c9a606bb00ea57c50fe27081273e0d5a8bcee Mon Sep 17 00:00:00 2001 From: Yinan Li Date: Wed, 13 Dec 2017 16:23:46 -0800 Subject: [PATCH 2/3] Fixed a unit test --- .../test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 35594ec47c94..2eb8a1fee104 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -408,7 +408,7 @@ class SparkSubmitSuite childArgsMap.get("--arg") should be (Some("arg1")) mainClass should be (KUBERNETES_CLUSTER_SUBMIT_CLASS) classpath should have length (0) - conf.get("spark.master") should be ("k8s:https://host:port") + conf.get("spark.master") should be ("k8s://https://host:port") conf.get("spark.executor.memory") should be ("5g") conf.get("spark.driver.memory") should be ("4g") conf.get("spark.kubernetes.namespace") should be ("spark") From a2e6396e4b7e1654aac0489b67777090cf59c4dc Mon Sep 17 00:00:00 2001 From: Yinan Li Date: Thu, 14 Dec 2017 08:32:39 -0800 Subject: [PATCH 3/3] Addressed comments --- .../services/org.apache.spark.scheduler.ExternalClusterManager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager b/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager index 7965fb133611..81d14766ffb8 100644 --- a/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager +++ b/resource-managers/kubernetes/core/src/main/resources/META-INF/services/org.apache.spark.scheduler.ExternalClusterManager @@ -1 +1 @@ -org.apache.spark.scheduler.cluster.k8s.KubernetesClusterManager \ No newline at end of file +org.apache.spark.scheduler.cluster.k8s.KubernetesClusterManager