From ebcbf05b212937aad21742bf14e592ffc1b14383 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 24 Aug 2018 10:32:14 +0100 Subject: [PATCH 1/6] [SPARK-25222][K8S] Improve container status logging Actually log human readable container status information rather than dumping the raw status object returned by the K8S API --- .../k8s/submit/LoggingPodStatusWatcher.scala | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala index 173ac541626a..dc93345eff7a 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala @@ -121,16 +121,22 @@ private[k8s] class LoggingPodStatusWatcherImpl( .map(_.getImage) .mkString(", ")), ("phase", pod.getStatus.getPhase), - ("status", pod.getStatus.getContainerStatuses.toString) + ("status", pod.getStatus.getContainerStatuses.asScala.map { status => + Seq( + ("Container name", status.getName), + ("Container image", status.getImage)) ++ + containerStatusDescription(status) + }.map(p => formatPairsBundle(p, 2)).mkString("\n\n")) ) formatPairsBundle(details) } - private def formatPairsBundle(pairs: Seq[(String, String)]) = { + private def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) = { // Use more loggable format if value is null or empty + val indentStr = "\t" * indent pairs.map { - case (k, v) => s"\n\t $k: ${Option(v).filter(_.nonEmpty).getOrElse("N/A")}" + case (k, v) => s"\n$indentStr $k: ${Option(v).filter(_.nonEmpty).getOrElse("N/A")}" }.mkString("") } @@ -147,7 +153,7 @@ private[k8s] class LoggingPodStatusWatcherImpl( ("Container name", status.getName), ("Container image", status.getImage)) ++ containerStatusDescription(status) - }.map(formatPairsBundle).mkString("\n\n") + }.map(p => formatPairsBundle(p, 1)).mkString("\n\n") } private def containerStatusDescription( @@ -168,7 +174,10 @@ private[k8s] class LoggingPodStatusWatcherImpl( case terminated: ContainerStateTerminated => Seq( ("Container state", "Terminated"), - ("Exit code", terminated.getExitCode.toString)) + ("Container started at", formatTime(terminated.getStartedAt)), + ("Container finished at", formatTime(terminated.getFinishedAt)), + ("Exit code", terminated.getExitCode.toString), + ("Termination reason", terminated.getReason)) case unknown => throw new SparkException(s"Unexpected container status type ${unknown.getClass}.") }.getOrElse(Seq(("Container state", "N/A"))) From 842a0b3773d35c36e4c696b6b6f518d6f05b8934 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 24 Aug 2018 14:26:06 +0100 Subject: [PATCH 2/6] [SPARK-25222][K8S] Move pod & container logging to utils Moves the methods for logging pod and container statuses into the KubernetesUtils class so they can be reused elsewhere --- .../spark/deploy/k8s/KubernetesUtils.scala | 92 ++++++++++++++++++- .../k8s/submit/LoggingPodStatusWatcher.scala | 80 +--------------- 2 files changed, 92 insertions(+), 80 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 588cd9d40f9a..f9cac2a0188c 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -16,7 +16,11 @@ */ package org.apache.spark.deploy.k8s -import org.apache.spark.SparkConf +import scala.collection.JavaConverters._ + +import io.fabric8.kubernetes.api.model.{ContainerStateRunning, ContainerStateTerminated, ContainerStateWaiting, ContainerStatus, Pod, Time} + +import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.util.Utils private[spark] object KubernetesUtils { @@ -60,4 +64,90 @@ private[spark] object KubernetesUtils { } def parseMasterUrl(url: String): String = url.substring("k8s://".length) + + def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) = { + // Use more loggable format if value is null or empty + val indentStr = "\t" * indent + pairs.map { + case (k, v) => s"\n$indentStr $k: ${Option(v).filter(_.nonEmpty).getOrElse("N/A")}" + }.mkString("") + } + + /** + * Given a pod output a human readable representation of its state + * @param pod Pod + * @return Human readable pod state + */ + def formatPodState(pod: Pod): String = { + val details = Seq[(String, String)]( + // pod metadata + ("pod name", pod.getMetadata.getName), + ("namespace", pod.getMetadata.getNamespace), + ("labels", pod.getMetadata.getLabels.asScala.mkString(", ")), + ("pod uid", pod.getMetadata.getUid), + ("creation time", formatTime(pod.getMetadata.getCreationTimestamp)), + + // spec details + ("service account name", pod.getSpec.getServiceAccountName), + ("volumes", pod.getSpec.getVolumes.asScala.map(_.getName).mkString(", ")), + ("node name", pod.getSpec.getNodeName), + + // status + ("start time", formatTime(pod.getStatus.getStartTime)), + ("container images", + pod.getStatus.getContainerStatuses + .asScala + .map(_.getImage) + .mkString(", ")), + ("phase", pod.getStatus.getPhase), + ("status", pod.getStatus.getContainerStatuses.asScala.map { status => + Seq( + ("Container name", status.getName), + ("Container image", status.getImage)) ++ + containerStatusDescription(status) + }.map(p => formatPairsBundle(p, 2)).mkString("\n\n")) + ) + + formatPairsBundle(details) + } + + def containersDescription(p: Pod): String = { + p.getStatus.getContainerStatuses.asScala.map { status => + Seq( + ("Container name", status.getName), + ("Container image", status.getImage)) ++ + containerStatusDescription(status) + }.map(p => formatPairsBundle(p, 1)).mkString("\n\n") + } + + def containerStatusDescription(containerStatus: ContainerStatus) + : Seq[(String, String)] = { + val state = containerStatus.getState + Option(state.getRunning) + .orElse(Option(state.getTerminated)) + .orElse(Option(state.getWaiting)) + .map { + case running: ContainerStateRunning => + Seq( + ("Container state", "Running"), + ("Container started at", formatTime(running.getStartedAt))) + case waiting: ContainerStateWaiting => + Seq( + ("Container state", "Waiting"), + ("Pending reason", waiting.getReason)) + case terminated: ContainerStateTerminated => + Seq( + ("Container state", "Terminated"), + ("Container started at", formatTime(terminated.getStartedAt)), + ("Container finished at", formatTime(terminated.getFinishedAt)), + ("Exit code", terminated.getExitCode.toString), + ("Termination reason", terminated.getReason)) + case unknown => + throw new SparkException(s"Unexpected container status type ${unknown.getClass}.") + }.getOrElse(Seq(("Container state", "N/A"))) + } + + def formatTime(time: Time): String = { + if (time != null) time.getTime else "N/A" + } } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala index dc93345eff7a..60448e81f7ea 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala @@ -25,6 +25,7 @@ import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher} import io.fabric8.kubernetes.client.Watcher.Action import org.apache.spark.SparkException +import org.apache.spark.deploy.k8s.KubernetesUtils._ import org.apache.spark.internal.Logging import org.apache.spark.util.ThreadUtils @@ -99,47 +100,6 @@ private[k8s] class LoggingPodStatusWatcherImpl( scheduler.shutdown() } - private def formatPodState(pod: Pod): String = { - val details = Seq[(String, String)]( - // pod metadata - ("pod name", pod.getMetadata.getName), - ("namespace", pod.getMetadata.getNamespace), - ("labels", pod.getMetadata.getLabels.asScala.mkString(", ")), - ("pod uid", pod.getMetadata.getUid), - ("creation time", formatTime(pod.getMetadata.getCreationTimestamp)), - - // spec details - ("service account name", pod.getSpec.getServiceAccountName), - ("volumes", pod.getSpec.getVolumes.asScala.map(_.getName).mkString(", ")), - ("node name", pod.getSpec.getNodeName), - - // status - ("start time", formatTime(pod.getStatus.getStartTime)), - ("container images", - pod.getStatus.getContainerStatuses - .asScala - .map(_.getImage) - .mkString(", ")), - ("phase", pod.getStatus.getPhase), - ("status", pod.getStatus.getContainerStatuses.asScala.map { status => - Seq( - ("Container name", status.getName), - ("Container image", status.getImage)) ++ - containerStatusDescription(status) - }.map(p => formatPairsBundle(p, 2)).mkString("\n\n")) - ) - - formatPairsBundle(details) - } - - private def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) = { - // Use more loggable format if value is null or empty - val indentStr = "\t" * indent - pairs.map { - case (k, v) => s"\n$indentStr $k: ${Option(v).filter(_.nonEmpty).getOrElse("N/A")}" - }.mkString("") - } - override def awaitCompletion(): Unit = { podCompletedFuture.await() logInfo(pod.map { p => @@ -147,43 +107,5 @@ private[k8s] class LoggingPodStatusWatcherImpl( }.getOrElse("No containers were found in the driver pod.")) } - private def containersDescription(p: Pod): String = { - p.getStatus.getContainerStatuses.asScala.map { status => - Seq( - ("Container name", status.getName), - ("Container image", status.getImage)) ++ - containerStatusDescription(status) - }.map(p => formatPairsBundle(p, 1)).mkString("\n\n") - } - - private def containerStatusDescription( - containerStatus: ContainerStatus): Seq[(String, String)] = { - val state = containerStatus.getState - Option(state.getRunning) - .orElse(Option(state.getTerminated)) - .orElse(Option(state.getWaiting)) - .map { - case running: ContainerStateRunning => - Seq( - ("Container state", "Running"), - ("Container started at", formatTime(running.getStartedAt))) - case waiting: ContainerStateWaiting => - Seq( - ("Container state", "Waiting"), - ("Pending reason", waiting.getReason)) - case terminated: ContainerStateTerminated => - Seq( - ("Container state", "Terminated"), - ("Container started at", formatTime(terminated.getStartedAt)), - ("Container finished at", formatTime(terminated.getFinishedAt)), - ("Exit code", terminated.getExitCode.toString), - ("Termination reason", terminated.getReason)) - case unknown => - throw new SparkException(s"Unexpected container status type ${unknown.getClass}.") - }.getOrElse(Seq(("Container state", "N/A"))) - } - private def formatTime(time: Time): String = { - if (time != null) time.getTime else "N/A" - } } From dcb5b007a35a6543435f5d3b2aeb02c9eef2ab8d Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Fri, 24 Aug 2018 14:33:17 +0100 Subject: [PATCH 3/6] [SPARK-25222][K8S] Human readable container status on task failure Modifies ExecutorPodsLifecycleManager so that it uses the container status formatting methods as part of its output on task failure. It also avoids outputting null reasons and messages. --- .../deploy/k8s/submit/LoggingPodStatusWatcher.scala | 2 -- .../cluster/k8s/ExecutorPodsLifecycleManager.scala | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala index 60448e81f7ea..1889fe5eb3e9 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/LoggingPodStatusWatcher.scala @@ -106,6 +106,4 @@ private[k8s] class LoggingPodStatusWatcherImpl( s"Container final statuses:\n\n${containersDescription(p)}" }.getOrElse("No containers were found in the driver pod.")) } - - } diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala index b28d93990313..3d1fdf4e1158 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala @@ -24,6 +24,7 @@ import scala.collection.mutable import org.apache.spark.SparkConf import org.apache.spark.deploy.k8s.Config._ +import org.apache.spark.deploy.k8s.KubernetesUtils._ import org.apache.spark.internal.Logging import org.apache.spark.scheduler.ExecutorExited import org.apache.spark.util.Utils @@ -151,13 +152,15 @@ private[spark] class ExecutorPodsLifecycleManager( private def exitReasonMessage(podState: FinalPodState, execId: Long, exitCode: Int) = { val pod = podState.pod + val reason = Option(pod.getStatus.getReason) + val message = Option(pod.getStatus.getMessage) s""" |The executor with id $execId exited with exit code $exitCode. - |The API gave the following brief reason: ${pod.getStatus.getReason} - |The API gave the following message: ${pod.getStatus.getMessage} + |The API gave the following brief reason: ${reason.getOrElse("")} + |The API gave the following message: ${message.getOrElse("")} |The API gave the following container statuses: | - |${pod.getStatus.getContainerStatuses.asScala.map(_.toString).mkString("\n===\n")} + |${containersDescription(pod)} """.stripMargin } From 355e66db2d3abf5ea58dc5261f723fed9416123f Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Tue, 28 Aug 2018 11:21:41 +0100 Subject: [PATCH 4/6] [SPARK-25222][K8S] Address comments and update tests - Address PR comments about consistent formatting and method resuse - Update tests to check for new improved log format --- .../spark/deploy/k8s/KubernetesUtils.scala | 42 +++++++------------ .../ExecutorPodsLifecycleManagerSuite.scala | 9 ++-- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index f9cac2a0188c..39dcb10780d2 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -74,7 +74,7 @@ private[spark] object KubernetesUtils { } /** - * Given a pod output a human readable representation of its state + * Given a pod, output a human readable representation of its state * @param pod Pod * @return Human readable pod state */ @@ -94,30 +94,20 @@ private[spark] object KubernetesUtils { // status ("start time", formatTime(pod.getStatus.getStartTime)), - ("container images", - pod.getStatus.getContainerStatuses - .asScala - .map(_.getImage) - .mkString(", ")), ("phase", pod.getStatus.getPhase), - ("status", pod.getStatus.getContainerStatuses.asScala.map { status => - Seq( - ("Container name", status.getName), - ("Container image", status.getImage)) ++ - containerStatusDescription(status) - }.map(p => formatPairsBundle(p, 2)).mkString("\n\n")) + ("container status", containersDescription(pod, 2)) ) formatPairsBundle(details) } - def containersDescription(p: Pod): String = { + def containersDescription(p: Pod, indent: Int = 1): String = { p.getStatus.getContainerStatuses.asScala.map { status => Seq( - ("Container name", status.getName), - ("Container image", status.getImage)) ++ + ("container name", status.getName), + ("container image", status.getImage)) ++ containerStatusDescription(status) - }.map(p => formatPairsBundle(p, 1)).mkString("\n\n") + }.map(p => formatPairsBundle(p, indent)).mkString("\n\n") } def containerStatusDescription(containerStatus: ContainerStatus) @@ -129,22 +119,22 @@ private[spark] object KubernetesUtils { .map { case running: ContainerStateRunning => Seq( - ("Container state", "Running"), - ("Container started at", formatTime(running.getStartedAt))) + ("container state", "running"), + ("container started at", formatTime(running.getStartedAt))) case waiting: ContainerStateWaiting => Seq( - ("Container state", "Waiting"), - ("Pending reason", waiting.getReason)) + ("container state", "waiting"), + ("pending reason", waiting.getReason)) case terminated: ContainerStateTerminated => Seq( - ("Container state", "Terminated"), - ("Container started at", formatTime(terminated.getStartedAt)), - ("Container finished at", formatTime(terminated.getFinishedAt)), - ("Exit code", terminated.getExitCode.toString), - ("Termination reason", terminated.getReason)) + ("container state", "terminated"), + ("container started at", formatTime(terminated.getStartedAt)), + ("container finished at", formatTime(terminated.getFinishedAt)), + ("exit code", terminated.getExitCode.toString), + ("termination reason", terminated.getReason)) case unknown => throw new SparkException(s"Unexpected container status type ${unknown.getClass}.") - }.getOrElse(Seq(("Container state", "N/A"))) + }.getOrElse(Seq(("container state", "N/A"))) } def formatTime(time: Time): String = { diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala index 562ace9f49d4..9e7f3d4ea4a2 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala @@ -31,6 +31,7 @@ import scala.collection.mutable import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.deploy.k8s.Fabric8Aliases._ +import org.apache.spark.deploy.k8s.KubernetesUtils._ import org.apache.spark.scheduler.ExecutorExited import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils._ @@ -104,13 +105,15 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte } private def exitReasonMessage(failedExecutorId: Int, failedPod: Pod): String = { + val reason = Option(failedPod.getStatus.getReason) + val message = Option(failedPod.getStatus.getMessage) s""" |The executor with id $failedExecutorId exited with exit code 1. - |The API gave the following brief reason: ${failedPod.getStatus.getReason} - |The API gave the following message: ${failedPod.getStatus.getMessage} + |The API gave the following brief reason: ${reason.getOrElse("")} + |The API gave the following message: ${message.getOrElse("")} |The API gave the following container statuses: | - |${failedPod.getStatus.getContainerStatuses.asScala.map(_.toString).mkString("\n===\n")} + |${containersDescription(failedPod)} """.stripMargin } From 6f6442f392717fe87002e9bc1b27c91ff387080e Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Wed, 29 Aug 2018 17:38:04 +0100 Subject: [PATCH 5/6] [SPARK-25222][K8S] Fix scalastyle for utils methods --- .../org/apache/spark/deploy/k8s/KubernetesUtils.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala index 39dcb10780d2..f5fae7cc8c47 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesUtils.scala @@ -65,7 +65,7 @@ private[spark] object KubernetesUtils { def parseMasterUrl(url: String): String = url.substring("k8s://".length) - def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) = { + def formatPairsBundle(pairs: Seq[(String, String)], indent: Int = 1) : String = { // Use more loggable format if value is null or empty val indentStr = "\t" * indent pairs.map { @@ -74,10 +74,11 @@ private[spark] object KubernetesUtils { } /** - * Given a pod, output a human readable representation of its state - * @param pod Pod - * @return Human readable pod state - */ + * Given a pod, output a human readable representation of its state + * + * @param pod Pod + * @return Human readable pod state + */ def formatPodState(pod: Pod): String = { val details = Seq[(String, String)]( // pod metadata From 4c39a81cd93ea0a7e8ccfe51558d9756f3ea7ff3 Mon Sep 17 00:00:00 2001 From: Rob Vesse Date: Mon, 3 Sep 2018 09:39:32 +0100 Subject: [PATCH 6/6] [SPARK-25222][K8S] Use N/A inside of empty When the API supplies no reason/message use N/A instead of the empty string --- .../scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala | 4 ++-- .../cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala index 3d1fdf4e1158..e2800cff7b72 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManager.scala @@ -156,8 +156,8 @@ private[spark] class ExecutorPodsLifecycleManager( val message = Option(pod.getStatus.getMessage) s""" |The executor with id $execId exited with exit code $exitCode. - |The API gave the following brief reason: ${reason.getOrElse("")} - |The API gave the following message: ${message.getOrElse("")} + |The API gave the following brief reason: ${reason.getOrElse("N/A")} + |The API gave the following message: ${message.getOrElse("N/A")} |The API gave the following container statuses: | |${containersDescription(pod)} diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala index 9e7f3d4ea4a2..d8409383b4a1 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsLifecycleManagerSuite.scala @@ -109,8 +109,8 @@ class ExecutorPodsLifecycleManagerSuite extends SparkFunSuite with BeforeAndAfte val message = Option(failedPod.getStatus.getMessage) s""" |The executor with id $failedExecutorId exited with exit code 1. - |The API gave the following brief reason: ${reason.getOrElse("")} - |The API gave the following message: ${message.getOrElse("")} + |The API gave the following brief reason: ${reason.getOrElse("N/A")} + |The API gave the following message: ${message.getOrElse("N/A")} |The API gave the following container statuses: | |${containersDescription(failedPod)}