From 67813e1c3d3269e28b6ac903143e87ddafd0ce6c Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 14 Mar 2017 14:45:12 -0700 Subject: [PATCH 1/8] Add job group page --- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- .../apache/spark/ui/jobs/AllJobsPage.scala | 4 +- .../apache/spark/ui/jobs/JobGroupPage.scala | 117 ++++++++++++++++++ .../org/apache/spark/ui/jobs/JobsTab.scala | 1 + docker-integration-tests/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml | 2 +- external/mqtt-assembly/pom.xml | 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml | 2 +- extras/kinesis-asl-assembly/pom.xml | 2 +- extras/kinesis-asl/pom.xml | 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib/pom.xml | 2 +- network/common/pom.xml | 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 2 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tags/pom.xml | 2 +- tools/pom.xml | 2 +- unsafe/pom.xml | 2 +- yarn/pom.xml | 2 +- 38 files changed, 155 insertions(+), 37 deletions(-) create mode 100644 core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala diff --git a/assembly/pom.xml b/assembly/pom.xml index 91862ef367f8..a4932dc1533d 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/bagel/pom.xml b/bagel/pom.xml index 0886aca416e3..445fb43e9536 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 986d17aac398..111024c7cbb1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 08dc17d5887e..74be395f8697 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -160,7 +160,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { events.toSeq } - private def makeTimeline( + protected def makeTimeline( jobs: Seq[JobUIData], executors: HashMap[String, ExecutorUIData], startTime: Long): Seq[Node] = { @@ -205,7 +205,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { } - private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { + protected def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined) val columns: Seq[Node] = { diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala new file mode 100644 index 000000000000..7b2b96eba2b2 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -0,0 +1,117 @@ +package org.apache.spark.ui.jobs + +import javax.servlet.http.HttpServletRequest + +import org.apache.spark.JobExecutionStatus +import org.apache.spark.ui.jobs.UIData.JobUIData +import org.apache.spark.ui.{UIUtils, WebUIPage} + +import scala.collection.mutable +import scala.xml.{Node, NodeSeq} + +/** Page showing list of jobs under a job group id */ +private[ui] class JobGroupPage(parent: JobsTab) extends AllJobsPage(parent) { + + override def render(request: HttpServletRequest): Seq[Node] = { + val listener = parent.jobProgresslistener + listener.synchronized { + val parameterId = request.getParameter("id") + require(parameterId != null && parameterId.nonEmpty, "Missing parameter id") + + val jobGroupId = parameterId + val groupToJobsTable = listener.jobGroupToJobIds.get(jobGroupId) + if (groupToJobsTable.isEmpty) { + val content = +
+

No information to display for jobGroup + {jobGroupId} +

+
+ return UIUtils.headerSparkPage( + s"Details for JobGroup $jobGroupId", content, parent) + } + + val jobsInGroup = listener.jobIdToData + + val activeJobsInGroup = mutable.Buffer[JobUIData]() + val completedJobsInGroup = mutable.Buffer[JobUIData]() + val failedJobsInGroup = mutable.Buffer[JobUIData]() + groupToJobsTable.get.foreach { jobId => + val job = jobsInGroup.get(jobId) + job.get.status match { + case JobExecutionStatus.RUNNING => activeJobsInGroup ++= job + case JobExecutionStatus.SUCCEEDED => completedJobsInGroup ++= job + case JobExecutionStatus.FAILED => failedJobsInGroup ++= job + } + } + + val activeJobsTable = + jobsTable(activeJobsInGroup.sortBy(_.submissionTime.getOrElse((-1L))).reverse) + val completedJobsTable = + jobsTable(completedJobsInGroup.sortBy(_.completionTime.getOrElse(-1L)).reverse) + val failedJobsTable = + jobsTable(failedJobsInGroup.sortBy(_.completionTime.getOrElse(-1L)).reverse) + + val shouldShowActiveJobs = activeJobsInGroup.nonEmpty + val shouldShowCompletedJobs = completedJobsInGroup.nonEmpty + val shouldShowFailedJobs = failedJobsInGroup.nonEmpty + + val summary: NodeSeq = +
+ +
+ + var content = summary + val executorListener = parent.executorListener + content ++= makeTimeline(activeJobsInGroup ++ completedJobsInGroup ++ failedJobsInGroup, + executorListener.executorIdToData, listener.startTime) + + if (shouldShowActiveJobs) { + content ++=

Active Jobs ( + {activeJobsInGroup.size} + )

++ + activeJobsTable + } + if (shouldShowCompletedJobs) { + content ++=

Completed Jobs ( + {completedJobsInGroup.size} + )

++ + completedJobsTable + } + if (shouldShowFailedJobs) { + content ++=

Failed Jobs ( + {failedJobsInGroup.size} + )

++ + failedJobsTable + } + + val helpText = + """A job is triggered by an action, like count() or saveAsTextFile().""" + + " Click on a job to see information about the stages of tasks inside it." + + UIUtils.headerSparkPage("Spark Jobs", content, parent, helpText = Some(helpText)) + } + + } + +} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala index 77ca60b000a9..3c56d9e026c0 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala @@ -33,4 +33,5 @@ private[ui] class JobsTab(parent: SparkUI) extends SparkUITab(parent, "jobs") { attachPage(new AllJobsPage(this)) attachPage(new JobPage(this)) + attachPage(new JobGroupPage(this)) } diff --git a/docker-integration-tests/pom.xml b/docker-integration-tests/pom.xml index 373020a9e0e0..e1af236bfef4 100644 --- a/docker-integration-tests/pom.xml +++ b/docker-integration-tests/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index f3f72be2ca78..26365698885f 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 43480bcd90dd..4fb5bfe31983 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 8ef26d37f6b2..6c48d2c1a8f5 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index e829098d44ce..870e4c8889c0 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index aa8b8db2307f..8ab6e3d1568c 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index f0383b2f224c..1316a65b88b3 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/mqtt-assembly/pom.xml b/external/mqtt-assembly/pom.xml index 7acc4702c296..64d606262926 100644 --- a/external/mqtt-assembly/pom.xml +++ b/external/mqtt-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 0d679fe5bbb1..6146833e3cf1 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 3052b46ed9b2..077ca99bd50a 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index c40368c9104f..180f506b4f0b 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index f2578b0ec7a8..15f1c239c781 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-1-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl-assembly/pom.xml b/extras/kinesis-asl-assembly/pom.xml index 68d4bb44b870..e8da0359f983 100644 --- a/extras/kinesis-asl-assembly/pom.xml +++ b/extras/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-1-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 6313d25aba25..1eb931315bec 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-1-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml index c3bc9eb2f828..124cbfeb4964 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/extras/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-1-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 25ad425e0852..17b4b46e19cf 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 20aeccfb2b69..d9f49465b83d 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 6cfcb1b41a0b..34f75f697621 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/network/common/pom.xml b/network/common/pom.xml index 0d05f32a929d..db0b60bf5cc3 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index f7a1d86ca6b8..de07eea626e9 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 6cf70d1b9dd5..67e92f786964 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index d6b660991e5c..48375a0b6c3c 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT pom Spark Project Parent POM http://spark.apache.org/ diff --git a/repl/pom.xml b/repl/pom.xml index 011a5cad39df..7a82f34e1e79 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index b6f461a440a1..2cf1641c486b 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index a1125e29035e..064ce0145a01 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 0942e06ebb1f..243aac62dcc7 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 6cbfe1192652..1028b2bf0b69 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 991abdfa2411..77831e7ebd0d 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/tags/pom.xml b/tags/pom.xml index ea8dfdbdd577..28cad3346559 100644 --- a/tags/pom.xml +++ b/tags/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 3dc0aa3766dd..50e099113b47 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 24095b721193..106654110b7b 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index 09ab458b33a5..14dda647c145 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-6-SNAPSHOT ../pom.xml From 805011684dfe293e722477fb9bb77f929229bdea Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 14 Mar 2017 20:21:40 -0700 Subject: [PATCH 2/8] bump up version and add more logic for show jobs by job group id --- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- .../apache/spark/ui/jobs/AllJobsPage.scala | 8 +- .../apache/spark/ui/jobs/JobGroupPage.scala | 261 +++++++++++++++++- docker-integration-tests/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml | 2 +- external/mqtt-assembly/pom.xml | 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- graphx/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib/pom.xml | 2 +- network/common/pom.xml | 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 4 +- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tags/pom.xml | 2 +- tools/pom.xml | 2 +- unsafe/pom.xml | 2 +- yarn/pom.xml | 2 +- 33 files changed, 295 insertions(+), 38 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index a4932dc1533d..334c0acda74e 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/bagel/pom.xml b/bagel/pom.xml index 445fb43e9536..d5bcbdc505d5 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 111024c7cbb1..220e54cfb06c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 74be395f8697..0503d1a0607e 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -205,7 +205,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { } - protected def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { + private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined) val columns: Seq[Node] = { @@ -231,9 +231,13 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { val jobDescription = UIUtils.makeDescription(lastStageDescription, basePathUri) val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId) + val groupUrl = "%s/jobs/jobgroup?id=%s".format(basePathUri, job.jobGroup.getOrElse("")) - {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")} + {job.jobId} + + {job.jobGroup.map(id => s"($id)").getOrElse("")} + {jobDescription} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala index 7b2b96eba2b2..313a1045873c 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -1,16 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.ui.jobs +import java.util.Date import javax.servlet.http.HttpServletRequest +import org.apache.commons.lang3.StringEscapeUtils import org.apache.spark.JobExecutionStatus -import org.apache.spark.ui.jobs.UIData.JobUIData -import org.apache.spark.ui.{UIUtils, WebUIPage} +import org.apache.spark.ui.jobs.UIData.{ExecutorUIData, JobUIData} +import org.apache.spark.ui.{ToolTips, UIUtils, WebUIPage} import scala.collection.mutable -import scala.xml.{Node, NodeSeq} +import scala.collection.mutable.{HashMap, ListBuffer} +import scala.xml.{Node, NodeSeq, Unparsed, Utility} /** Page showing list of jobs under a job group id */ -private[ui] class JobGroupPage(parent: JobsTab) extends AllJobsPage(parent) { +private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { + private val JOBS_LEGEND = +
+ + Succeeded + + Failed + + Running +
.toString.filter(_ != '\n') + + private val EXECUTORS_LEGEND = +
+ + Added + + Removed +
.toString.filter(_ != '\n') + + private def getLastStageNameAndDescription(job: JobUIData): (String, String) = { + val lastStageInfo = Option(job.stageIds) + .filter(_.nonEmpty) + .flatMap { ids => parent.jobProgresslistener.stageIdToInfo.get(ids.max)} + val lastStageData = lastStageInfo.flatMap { s => + parent.jobProgresslistener.stageIdToData.get((s.stageId, s.attemptId)) + } + val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)") + val description = lastStageData.flatMap(_.description).getOrElse("") + (name, description) + } + + private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { + val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined) + + val columns: Seq[Node] = { + {if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"} + Description + Submitted + Duration + Stages: Succeeded/Total + Tasks (for all stages): Succeeded/Total + } + + def makeRow(job: JobUIData): Seq[Node] = { + val (lastStageName, lastStageDescription) = getLastStageNameAndDescription(job) + val duration: Option[Long] = { + job.submissionTime.map { start => + val end = job.completionTime.getOrElse(System.currentTimeMillis()) + end - start + } + } + val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") + val formattedSubmissionTime = job.submissionTime.map(UIUtils.formatDate).getOrElse("Unknown") + val basePathUri = UIUtils.prependBaseUri(parent.basePath) + val jobDescription = UIUtils.makeDescription(lastStageDescription, basePathUri) + + val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId) + + + {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")} + + + {jobDescription} + {lastStageName} + + + {formattedSubmissionTime} + + {formattedDuration} + + {job.completedStageIndices.size}/{job.stageIds.size - job.numSkippedStages} + {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"} + {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"} + + + {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks, + failed = job.numFailedTasks, skipped = job.numSkippedTasks, + total = job.numTasks - job.numSkippedTasks)} + + + } + + + {columns} + + {jobs.map(makeRow)} + +
+ } + + private def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = { + jobUIDatas.filter { jobUIData => + jobUIData.status != JobExecutionStatus.UNKNOWN && jobUIData.submissionTime.isDefined + }.map { jobUIData => + val jobId = jobUIData.jobId + val status = jobUIData.status + val (jobName, jobDescription) = getLastStageNameAndDescription(jobUIData) + val displayJobDescription = if (jobDescription.isEmpty) jobName else jobDescription + val submissionTime = jobUIData.submissionTime.get + val completionTimeOpt = jobUIData.completionTime + val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis()) + val classNameByStatus = status match { + case JobExecutionStatus.SUCCEEDED => "succeeded" + case JobExecutionStatus.FAILED => "failed" + case JobExecutionStatus.RUNNING => "running" + case JobExecutionStatus.UNKNOWN => "unknown" + } + + // The timeline library treats contents as HTML, so we have to escape them. We need to add + // extra layers of escaping in order to embed this in a Javascript string literal. + val escapedDesc = Utility.escape(displayJobDescription) + val jsEscapedDesc = StringEscapeUtils.escapeEcmaScript(escapedDesc) + val jobEventJsonAsStr = + s""" + |{ + | 'className': 'job application-timeline-object ${classNameByStatus}', + | 'group': 'jobs', + | 'start': new Date(${submissionTime}), + | 'end': new Date(${completionTime}), + | 'content': '
Completed: ${UIUtils.formatDate(new Date(completionTime))}""" + } else { + "" + } + }">' + + | '${jsEscapedDesc} (Job ${jobId})
' + |} + """.stripMargin + jobEventJsonAsStr + } + } + + private def makeExecutorEvent(executorUIDatas: HashMap[String, ExecutorUIData]): Seq[String] = { + val events = ListBuffer[String]() + executorUIDatas.foreach { + case (executorId, event) => + val addedEvent = + s""" + |{ + | 'className': 'executor added', + | 'group': 'executors', + | 'start': new Date(${event.startTime}), + | 'content': '
Executor ${executorId} added
' + |} + """.stripMargin + events += addedEvent + + if (event.finishTime.isDefined) { + val removedEvent = + s""" + |{ + | 'className': 'executor removed', + | 'group': 'executors', + | 'start': new Date(${event.finishTime.get}), + | 'content': '
Reason: ${event.finishReason.get.replace("\n", " ")}""" + } else { + "" + } + }"' + + | 'data-html="true">Executor ${executorId} removed
' + |} + """.stripMargin + events += removedEvent + } + } + events.toSeq + } + + + protected def makeTimeline( + jobs: Seq[JobUIData], + executors: HashMap[String, ExecutorUIData], + startTime: Long): Seq[Node] = { + + val jobEventJsonAsStrSeq = makeJobEvent(jobs) + val executorEventJsonAsStrSeq = makeExecutorEvent(executors) + + val groupJsonArrayAsStr = + s""" + |[ + | { + | 'id': 'executors', + | 'content': '
Executors
${EXECUTORS_LEGEND}', + | }, + | { + | 'id': 'jobs', + | 'content': '
Jobs
${JOBS_LEGEND}', + | } + |] + """.stripMargin + + val eventArrayAsStr = + (jobEventJsonAsStrSeq ++ executorEventJsonAsStrSeq).mkString("[", ",", "]") + + + + + Event Timeline + + ++ + ++ + + } override def render(request: HttpServletRequest): Seq[Node] = { val listener = parent.jobProgresslistener diff --git a/docker-integration-tests/pom.xml b/docker-integration-tests/pom.xml index e1af236bfef4..440a0f8b8d34 100644 --- a/docker-integration-tests/pom.xml +++ b/docker-integration-tests/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index 26365698885f..922058aa80d2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 4fb5bfe31983..8cfbd6eaa6be 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index 6c48d2c1a8f5..e70d1a92617f 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 870e4c8889c0..870dbc4fcadb 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 8ab6e3d1568c..21b71904cc29 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index 1316a65b88b3..cd326f357241 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/mqtt-assembly/pom.xml b/external/mqtt-assembly/pom.xml index 64d606262926..366725a88c02 100644 --- a/external/mqtt-assembly/pom.xml +++ b/external/mqtt-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 6146833e3cf1..15fa7e6abc13 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 077ca99bd50a..838db00fd919 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 180f506b4f0b..4680a0e288d7 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 17b4b46e19cf..3e3a136252a1 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index d9f49465b83d..3b83a3111768 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 34f75f697621..5b3fc7748e74 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/network/common/pom.xml b/network/common/pom.xml index db0b60bf5cc3..18cdf2d2bb9d 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index de07eea626e9..5a9a2e351030 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 67e92f786964..6fe5e8a40ee3 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/pom.xml b/pom.xml index 48375a0b6c3c..548e65399f30 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 pom Spark Project Parent POM http://spark.apache.org/ @@ -40,7 +40,7 @@ scm:git:git@github.com:clearstorydata/spark.git scm:git:git@github.com:clearstorydata/spark.git scm:git:git@github.com:clearstorydata/spark.git - HEAD + spark-parent_2.11-1.6.3-csd-6 diff --git a/repl/pom.xml b/repl/pom.xml index 7a82f34e1e79..c57713997efb 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 2cf1641c486b..48815b306f08 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 064ce0145a01..e3a4a820dcf4 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 243aac62dcc7..209e8b24a228 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 1028b2bf0b69..81b66ea8c364 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index 77831e7ebd0d..fe3d098175ac 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/tags/pom.xml b/tags/pom.xml index 28cad3346559..cb0cad57878d 100644 --- a/tags/pom.xml +++ b/tags/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index 50e099113b47..cdadd5ced7c8 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 106654110b7b..584996a38a98 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index 14dda647c145..d525eb992217 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-6 ../pom.xml From cf78a32c9d8fd75b14d8f5f0f46c22fbacc34540 Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 14 Mar 2017 21:09:32 -0700 Subject: [PATCH 3/8] Add total duration --- .../apache/spark/ui/jobs/JobGroupPage.scala | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala index 313a1045873c..343ebefd7a7b 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -289,15 +289,29 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { val activeJobsInGroup = mutable.Buffer[JobUIData]() val completedJobsInGroup = mutable.Buffer[JobUIData]() val failedJobsInGroup = mutable.Buffer[JobUIData]() + var totalDuration = 0L groupToJobsTable.get.foreach { jobId => - val job = jobsInGroup.get(jobId) - job.get.status match { + val job = jobsInGroup.get(jobId).get + val duration: Option[Long] = { + job.submissionTime.map { start => + val end = job.completionTime.getOrElse(System.currentTimeMillis()) + end - start + } + } + totalDuration += duration.getOrElse(0L) + job.status match { case JobExecutionStatus.RUNNING => activeJobsInGroup ++= job case JobExecutionStatus.SUCCEEDED => completedJobsInGroup ++= job case JobExecutionStatus.FAILED => failedJobsInGroup ++= job } } + val formattedDuration = if (totalDuration == 0) { + "Unknown" + } else { + UIUtils.formatDuration(totalDuration) + } + val activeJobsTable = jobsTable(activeJobsInGroup.sortBy(_.submissionTime.getOrElse((-1L))).reverse) val completedJobsTable = @@ -312,6 +326,14 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { val summary: NodeSeq =
From b46fc8d6754c2900aa6657c14beea6c46f4b775e Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 14 Mar 2017 21:26:14 -0700 Subject: [PATCH 4/8] Fix job status --- .../scala/org/apache/spark/ui/jobs/JobGroupPage.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala index 343ebefd7a7b..103ac79ef134 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -291,15 +291,15 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { val failedJobsInGroup = mutable.Buffer[JobUIData]() var totalDuration = 0L groupToJobsTable.get.foreach { jobId => - val job = jobsInGroup.get(jobId).get + val job = jobsInGroup.get(jobId) val duration: Option[Long] = { - job.submissionTime.map { start => - val end = job.completionTime.getOrElse(System.currentTimeMillis()) + job.get.submissionTime.map { start => + val end = job.get.completionTime.getOrElse(System.currentTimeMillis()) end - start } } totalDuration += duration.getOrElse(0L) - job.status match { + job.get.status match { case JobExecutionStatus.RUNNING => activeJobsInGroup ++= job case JobExecutionStatus.SUCCEEDED => completedJobsInGroup ++= job case JobExecutionStatus.FAILED => failedJobsInGroup ++= job From 63762c7e5ab2b0276bf30522a5639ae6c4cd13ef Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Fri, 24 Mar 2017 13:48:29 -0700 Subject: [PATCH 5/8] Uncommit the pom version bump --- assembly/pom.xml | 2 +- bagel/pom.xml | 2 +- core/pom.xml | 2 +- docker-integration-tests/pom.xml | 2 +- examples/pom.xml | 2 +- external/flume-assembly/pom.xml | 2 +- external/flume-sink/pom.xml | 2 +- external/flume/pom.xml | 2 +- external/kafka-assembly/pom.xml | 2 +- external/kafka/pom.xml | 2 +- external/mqtt-assembly/pom.xml | 2 +- external/mqtt/pom.xml | 2 +- external/twitter/pom.xml | 2 +- external/zeromq/pom.xml | 2 +- extras/java8-tests/pom.xml | 2 +- extras/kinesis-asl-assembly/pom.xml | 2 +- extras/kinesis-asl/pom.xml | 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- graphx/pom.xml | 2 +- launcher/pom.xml | 2 +- mllib/pom.xml | 2 +- network/common/pom.xml | 2 +- network/shuffle/pom.xml | 2 +- network/yarn/pom.xml | 2 +- pom.xml | 4 ++-- repl/pom.xml | 2 +- sql/catalyst/pom.xml | 2 +- sql/core/pom.xml | 2 +- sql/hive-thriftserver/pom.xml | 2 +- sql/hive/pom.xml | 2 +- streaming/pom.xml | 2 +- tags/pom.xml | 2 +- tools/pom.xml | 2 +- unsafe/pom.xml | 2 +- yarn/pom.xml | 2 +- 35 files changed, 36 insertions(+), 36 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index 334c0acda74e..91862ef367f8 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/bagel/pom.xml b/bagel/pom.xml index d5bcbdc505d5..0886aca416e3 100644 --- a/bagel/pom.xml +++ b/bagel/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/core/pom.xml b/core/pom.xml index 220e54cfb06c..986d17aac398 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/docker-integration-tests/pom.xml b/docker-integration-tests/pom.xml index 440a0f8b8d34..373020a9e0e0 100644 --- a/docker-integration-tests/pom.xml +++ b/docker-integration-tests/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/examples/pom.xml b/examples/pom.xml index 922058aa80d2..f3f72be2ca78 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml index 8cfbd6eaa6be..43480bcd90dd 100644 --- a/external/flume-assembly/pom.xml +++ b/external/flume-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml index e70d1a92617f..8ef26d37f6b2 100644 --- a/external/flume-sink/pom.xml +++ b/external/flume-sink/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/flume/pom.xml b/external/flume/pom.xml index 870dbc4fcadb..e829098d44ce 100644 --- a/external/flume/pom.xml +++ b/external/flume/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml index 21b71904cc29..aa8b8db2307f 100644 --- a/external/kafka-assembly/pom.xml +++ b/external/kafka-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml index cd326f357241..f0383b2f224c 100644 --- a/external/kafka/pom.xml +++ b/external/kafka/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/mqtt-assembly/pom.xml b/external/mqtt-assembly/pom.xml index 366725a88c02..7acc4702c296 100644 --- a/external/mqtt-assembly/pom.xml +++ b/external/mqtt-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml index 15fa7e6abc13..0d679fe5bbb1 100644 --- a/external/mqtt/pom.xml +++ b/external/mqtt/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml index 838db00fd919..3052b46ed9b2 100644 --- a/external/twitter/pom.xml +++ b/external/twitter/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml index 4680a0e288d7..c40368c9104f 100644 --- a/external/zeromq/pom.xml +++ b/external/zeromq/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index 15f1c239c781..df78bdb4edd2 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl-assembly/pom.xml b/extras/kinesis-asl-assembly/pom.xml index e8da0359f983..43bcbcf9f1da 100644 --- a/extras/kinesis-asl-assembly/pom.xml +++ b/extras/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 1eb931315bec..170cfacf3fa9 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml index 124cbfeb4964..6909814d366f 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/extras/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6-SNAPSHOT + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/graphx/pom.xml b/graphx/pom.xml index 3e3a136252a1..25ad425e0852 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/launcher/pom.xml b/launcher/pom.xml index 3b83a3111768..20aeccfb2b69 100644 --- a/launcher/pom.xml +++ b/launcher/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/mllib/pom.xml b/mllib/pom.xml index 5b3fc7748e74..6cfcb1b41a0b 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/network/common/pom.xml b/network/common/pom.xml index 18cdf2d2bb9d..0d05f32a929d 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index 5a9a2e351030..f7a1d86ca6b8 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 6fe5e8a40ee3..6cf70d1b9dd5 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 548e65399f30..d6b660991e5c 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT pom Spark Project Parent POM http://spark.apache.org/ @@ -40,7 +40,7 @@ scm:git:git@github.com:clearstorydata/spark.git scm:git:git@github.com:clearstorydata/spark.git scm:git:git@github.com:clearstorydata/spark.git - spark-parent_2.11-1.6.3-csd-6 + HEAD diff --git a/repl/pom.xml b/repl/pom.xml index c57713997efb..011a5cad39df 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml index 48815b306f08..b6f461a440a1 100644 --- a/sql/catalyst/pom.xml +++ b/sql/catalyst/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/sql/core/pom.xml b/sql/core/pom.xml index e3a4a820dcf4..a1125e29035e 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index 209e8b24a228..0942e06ebb1f 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 81b66ea8c364..6cbfe1192652 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../../pom.xml diff --git a/streaming/pom.xml b/streaming/pom.xml index fe3d098175ac..991abdfa2411 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/tags/pom.xml b/tags/pom.xml index cb0cad57878d..ea8dfdbdd577 100644 --- a/tags/pom.xml +++ b/tags/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/tools/pom.xml b/tools/pom.xml index cdadd5ced7c8..3dc0aa3766dd 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/unsafe/pom.xml b/unsafe/pom.xml index 584996a38a98..24095b721193 100644 --- a/unsafe/pom.xml +++ b/unsafe/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml diff --git a/yarn/pom.xml b/yarn/pom.xml index d525eb992217..09ab458b33a5 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-6 + 1.6.3-csd-5-SNAPSHOT ../pom.xml From af7796396472ade22511eee3cb9247ae3b503481 Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 4 Apr 2017 15:55:55 -0700 Subject: [PATCH 6/8] Revert unnecessary version changes. --- .../apache/spark/ui/jobs/AllJobsPage.scala | 2 +- .../apache/spark/ui/jobs/JobGroupPage.scala | 28 +++++++++---------- .../spark/launcher/SparkLauncherSuite.java | 1 + extras/java8-tests/pom.xml | 2 +- extras/kinesis-asl-assembly/pom.xml | 2 +- extras/kinesis-asl/pom.xml | 2 +- extras/spark-ganglia-lgpl/pom.xml | 2 +- 7 files changed, 20 insertions(+), 19 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 0503d1a0607e..4ca57715f87b 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -160,7 +160,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { events.toSeq } - protected def makeTimeline( + private def makeTimeline( jobs: Seq[JobUIData], executors: HashMap[String, ExecutorUIData], startTime: Long): Seq[Node] = { diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala index 103ac79ef134..98dc142bf8dc 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -160,12 +160,12 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { | 'Status: ${status}
' + | 'Submitted: ${UIUtils.formatDate(new Date(submissionTime))}' + | '${ - if (status != JobExecutionStatus.RUNNING) { - s"""
Completed: ${UIUtils.formatDate(new Date(completionTime))}""" - } else { - "" - } - }">' + + if (status != JobExecutionStatus.RUNNING) { + s"""
Completed: ${UIUtils.formatDate(new Date(completionTime))}""" + } else { + "" + } + }">' + | '${jsEscapedDesc} (Job ${jobId})
' |} """.stripMargin @@ -204,12 +204,12 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { | 'data-title="Executor ${executorId}
' + | 'Removed at ${UIUtils.formatDate(new Date(event.finishTime.get))}' + | '${ - if (event.finishReason.isDefined) { - s"""
Reason: ${event.finishReason.get.replace("\n", " ")}""" - } else { - "" - } - }"' + + if (event.finishReason.isDefined) { + s"""
Reason: ${event.finishReason.get.replace("\n", " ")}""" + } else { + "" + } + }"' + | 'data-html="true">Executor ${executorId} removed' |} """.stripMargin @@ -381,8 +381,8 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { } val helpText = - """A job is triggered by an action, like count() or saveAsTextFile().""" + - " Click on a job to see information about the stages of tasks inside it." + s"""A job is triggered by an action, like count() or saveAsTextFile(). + | Click on a job to see information about the stages of tasks inside it.""".stripMargin UIUtils.headerSparkPage("Spark Jobs", content, parent, helpText = Some(helpText)) } diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index aa15e792e2b2..99d610b7f43b 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -125,3 +125,4 @@ public static void main(String[] args) throws Exception { } } +f diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml index df78bdb4edd2..f2578b0ec7a8 100644 --- a/extras/java8-tests/pom.xml +++ b/extras/java8-tests/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-1-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl-assembly/pom.xml b/extras/kinesis-asl-assembly/pom.xml index 43bcbcf9f1da..68d4bb44b870 100644 --- a/extras/kinesis-asl-assembly/pom.xml +++ b/extras/kinesis-asl-assembly/pom.xml @@ -21,7 +21,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-1-SNAPSHOT ../../pom.xml diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml index 170cfacf3fa9..6313d25aba25 100644 --- a/extras/kinesis-asl/pom.xml +++ b/extras/kinesis-asl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-1-SNAPSHOT ../../pom.xml diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml index 6909814d366f..c3bc9eb2f828 100644 --- a/extras/spark-ganglia-lgpl/pom.xml +++ b/extras/spark-ganglia-lgpl/pom.xml @@ -20,7 +20,7 @@ org.apache.spark spark-parent_2.11 - 1.6.3-csd-5-SNAPSHOT + 1.6.3-csd-1-SNAPSHOT ../../pom.xml From b2222322e3ef8480d5ed2e283f52b5ba5a486450 Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Tue, 4 Apr 2017 15:59:06 -0700 Subject: [PATCH 7/8] Undo the typo. --- .../test/java/org/apache/spark/launcher/SparkLauncherSuite.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java index 99d610b7f43b..aa15e792e2b2 100644 --- a/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java +++ b/core/src/test/java/org/apache/spark/launcher/SparkLauncherSuite.java @@ -125,4 +125,3 @@ public static void main(String[] args) throws Exception { } } -f From b7307cb219950a5af63f4df5170cf08206b9bcac Mon Sep 17 00:00:00 2001 From: Yewei Zhang Date: Wed, 5 Apr 2017 14:26:58 -0700 Subject: [PATCH 8/8] Add tests for job group page --- .../apache/spark/ui/jobs/AllJobsPage.scala | 8 ++-- .../apache/spark/ui/jobs/JobGroupPage.scala | 46 ++++++++----------- .../org/apache/spark/ui/UISeleniumSuite.scala | 38 +++++++++++++++ 3 files changed, 61 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala index 4ca57715f87b..3bda2000b783 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala @@ -231,13 +231,13 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { val jobDescription = UIUtils.makeDescription(lastStageDescription, basePathUri) val detailUrl = "%s/jobs/job?id=%s".format(basePathUri, job.jobId) - val groupUrl = "%s/jobs/jobgroup?id=%s".format(basePathUri, job.jobGroup.getOrElse("")) - {job.jobId} - - {job.jobGroup.map(id => s"($id)").getOrElse("")} + {job.jobId}{ job.jobGroup.map { id => + + {id} + }.getOrElse({job.jobGroup.map(id => s"($id)").getOrElse("")})} {jobDescription} diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala index 98dc142bf8dc..c1ef0b38c187 100644 --- a/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobGroupPage.scala @@ -303,15 +303,10 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { case JobExecutionStatus.RUNNING => activeJobsInGroup ++= job case JobExecutionStatus.SUCCEEDED => completedJobsInGroup ++= job case JobExecutionStatus.FAILED => failedJobsInGroup ++= job + case JobExecutionStatus.UNKNOWN => // not handling unknown status } } - val formattedDuration = if (totalDuration == 0) { - "Unknown" - } else { - UIUtils.formatDuration(totalDuration) - } - val activeJobsTable = jobsTable(activeJobsInGroup.sortBy(_.submissionTime.getOrElse((-1L))).reverse) val completedJobsTable = @@ -327,30 +322,33 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") {
  • - Total Duration: - {formattedDuration} + Total Duration: + { + if (totalDuration == 0) { + "Unknown" + } else { + UIUtils.formatDuration(totalDuration) + } + }
  • - Total jobs submitted: + Total jobs submitted: {activeJobsInGroup.size + completedJobsInGroup.size + failedJobsInGroup.size}
    • {if (shouldShowActiveJobs) {
  • - - Active Jobs: - {activeJobsInGroup.size} + Active Jobs: + {activeJobsInGroup.size}
    • }}{if (shouldShowCompletedJobs) {
  • - - Completed Jobs: - {completedJobsInGroup.size} + Completed Jobs: + {completedJobsInGroup.size}
    • }}{if (shouldShowFailedJobs) {
  • - - Failed Jobs: - {failedJobsInGroup.size} + Failed Jobs: + {failedJobsInGroup.size}
    • }}
@@ -362,21 +360,15 @@ private[ui] class JobGroupPage(parent: JobsTab) extends WebUIPage("jobgroup") { executorListener.executorIdToData, listener.startTime) if (shouldShowActiveJobs) { - content ++=

Active Jobs ( - {activeJobsInGroup.size} - )

++ + content ++=

Active Jobs ({activeJobsInGroup.size})

++ activeJobsTable } if (shouldShowCompletedJobs) { - content ++=

Completed Jobs ( - {completedJobsInGroup.size} - )

++ + content ++=

Completed Jobs ({completedJobsInGroup.size})

++ completedJobsTable } if (shouldShowFailedJobs) { - content ++=

Failed Jobs ( - {failedJobsInGroup.size} - )

++ + content ++=

Failed Jobs ({failedJobsInGroup.size})

++ failedJobsTable } diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala index 36c484dcfe8c..61ea0f862d07 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala @@ -645,6 +645,44 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B } } + test("jobs with job group id should show correct link to job group page on all jobs page") { + withSpark(newSparkContext()) { sc => + val ui = sc.ui.get + // Once at least one job has been run in a job group, then we should display the link to + // job group page + val jobGroupId = "my-job-group" + sc.setJobGroup(jobGroupId, "my-job-group-description") + // Create an RDD that involves multiple stages: + val rdd = + sc.parallelize(Seq(1, 2, 3)).map(identity).groupBy(identity).map(identity).groupBy(identity) + // Run it twice; this will cause the second job to have two "phantom" stages that were + // mentioned in its job start event but which were never actually executed: + rdd.count() + eventually(timeout(10 seconds), interval(50 milliseconds)) { + goToUi(ui, "/jobs") + findAll(cssSelector("tbody tr")).foreach { row => + val links = row.underlying.findElements(By.xpath(".//a")) + links.size should be (2) + links.get(0).getText().toLowerCase should include (jobGroupId) + links.get(0).getAttribute("href") should include regex ( + s"(?=.*jobgroup)(?=.*${jobGroupId})") + links.get(1).getText().toLowerCase should include ("count") + } + } + + eventually(timeout(10 seconds), interval(50 milliseconds)) { + goToUi(ui, s"/jobs/jobgroup/?id=${jobGroupId}") + find(id("pending")) should be (None) + find(id("active")) should be (None) + find(id("failed")) should be (None) + find(id("completed")).get.text should be ("Completed Jobs (1)") + findAll(cssSelector("tbody tr a")).foreach { link => + link.text.toLowerCase should include ("count") + } + } + } + } + def goToUi(sc: SparkContext, path: String): Unit = { goToUi(sc.ui.get, path) }