Skip to content

Commit ee7a7f0

Browse files
sarutaksarutak
committed
Refactored
1 parent 6a91872 commit ee7a7f0

File tree

4 files changed

+127
-140
lines changed

4 files changed

+127
-140
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/timeline-view.js‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
2424
return a.value - b.value
2525
},
2626
editable: false,
27-
showCurrentTime: false,
28-
showCustomTime: true,
27+
showCurrentTime: true,
28+
min: startTime,
2929
zoomable: false
3030
};
3131

@@ -34,10 +34,6 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
3434
applicationTimeline.setGroups(groups);
3535
applicationTimeline.setItems(items);
3636

37-
if (startTime != -1) {
38-
applicationTimeline.setCustomTime(startTime);
39-
}
40-
4137
setupZoomable("#application-timeline-zoom-lock", applicationTimeline);
4238

4339
$(".item.range.job.application-timeline-object").each(function() {
@@ -49,7 +45,7 @@ function drawApplicationTimeline(groupArray, eventObjArray, startTime) {
4945
});
5046
}
5147

52-
function drawJobTimeline(groupArray, eventObjArray, startTime) {
48+
function drawJobTimeline(groupArray, eventObjArray) {
5349
var groups = new vis.DataSet(groupArray);
5450
var items = new vis.DataSet(eventObjArray);
5551
var container = $('#job-timeline')[0];
@@ -59,7 +55,6 @@ function drawJobTimeline(groupArray, eventObjArray, startTime) {
5955
},
6056
editable: false,
6157
showCurrentTime: false,
62-
showCustomTime: true,
6358
zoomable: false,
6459
};
6560

@@ -68,10 +63,6 @@ function drawJobTimeline(groupArray, eventObjArray, startTime) {
6863
jobTimeline.setGroups(groups);
6964
jobTimeline.setItems(items);
7065

71-
if (startTime != -1) {
72-
jobTimeline.setCustomTime(startTime)
73-
}
74-
7566
setupZoomable("#job-timeline-zoom-lock", jobTimeline);
7667

7768
$(".item.range.stage.job-timeline-object").each(function() {

‎core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala‎

Lines changed: 64 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@ import org.apache.spark.JobExecutionStatus
2929

3030
/** Page showing list of all ongoing and recently finished jobs */
3131
private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
32-
private val startTime: Option[Long] = parent.sc.map(_.startTime)
33-
private val listener = parent.listener
3432
private val JOBS_LEGEND =
3533
<div class="legend-area"><svg width="200px" height="85px">
3634
<rect x="5px" y="5px" width="20px" height="15px"
@@ -44,41 +42,44 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
4442
<text x="35px" y="77px">Running Job</text>
4543
</svg></div>.toString.filter(_ != '\n')
4644

45+
private val EXECUTORS_LEGEND =
46+
<div class="legend-area"><svg width="200px" height="55px">
47+
<rect x="5px" y="5px" width="20px" height="15px"
48+
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
49+
<text x="35px" y="17px">Executor Added</text>
50+
<rect x="5px" y="35px" width="20px" height="15px"
51+
rx="2px" ry="2px" stroke="#97B0F8" fill="#EBCA59"></rect>
52+
<text x="35px" y="47px">Executor Removed</text>
53+
</svg></div>.toString.filter(_ != '\n')
54+
4755
private def getlastStageDescription(job: JobUIData) = {
4856
val lastStageInfo = Option(job.stageIds)
4957
.filter(_.nonEmpty)
50-
.flatMap { ids => listener.stageIdToInfo.get(ids.max)}
58+
.flatMap { ids => parent.listener.stageIdToInfo.get(ids.max)}
5159
val lastStageData = lastStageInfo.flatMap { s =>
52-
listener.stageIdToData.get((s.stageId, s.attemptId))
60+
parent.listener.stageIdToData.get((s.stageId, s.attemptId))
5361
}
5462
lastStageData.flatMap(_.description).getOrElse("")
5563
}
5664

57-
private def makeTimeline(jobs: Seq[JobUIData], executors: Seq[ExecutorUIData]): Seq[Node] = {
58-
59-
def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = {
60-
jobUIDatas.flatMap { jobUIData =>
61-
val jobId = jobUIData.jobId
62-
val status = jobUIData.status
63-
val jobDescription = getlastStageDescription(jobUIData)
64-
val submissionTimeOpt = jobUIData.submissionTime
65-
val completionTimeOpt = jobUIData.completionTime
66-
67-
if (status == JobExecutionStatus.UNKNOWN || submissionTimeOpt.isEmpty ||
68-
completionTimeOpt.isEmpty && status != JobExecutionStatus.RUNNING) {
69-
None
70-
}
71-
72-
val submissionTime = submissionTimeOpt.get
73-
val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis())
74-
val classNameByStatus = status match {
75-
case JobExecutionStatus.SUCCEEDED => "succeeded"
76-
case JobExecutionStatus.FAILED => "failed"
77-
case JobExecutionStatus.RUNNING => "running"
78-
}
65+
private def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = {
66+
jobUIDatas.filter { jobUIData =>
67+
jobUIData.status != JobExecutionStatus.UNKNOWN && jobUIData.submissionTime.isDefined
68+
}.map { jobUIData =>
69+
val jobId = jobUIData.jobId
70+
val status = jobUIData.status
71+
val jobDescription = getlastStageDescription(jobUIData)
72+
val submissionTime = jobUIData.submissionTime.get
73+
val completionTimeOpt = jobUIData.completionTime
74+
val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis())
75+
val classNameByStatus = status match {
76+
case JobExecutionStatus.SUCCEEDED => "succeeded"
77+
case JobExecutionStatus.FAILED => "failed"
78+
case JobExecutionStatus.RUNNING => "running"
79+
}
7980

80-
val jobEventJsonAsStr =
81-
s"""
81+
val jobEventJsonAsStr =
82+
s"""
8283
|{
8384
| 'className': 'job application-timeline-object ${classNameByStatus}',
8485
| 'group': 'jobs',
@@ -89,25 +90,25 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
8990
| 'title': '${jobDescription} (Job ${jobId})\\nStatus: ${status}\\n' +
9091
| 'Submission Time: ${UIUtils.formatDate(new Date(submissionTime))}' +
9192
| '${
92-
if (status != JobExecutionStatus.RUNNING) {
93-
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
94-
} else {
95-
""
96-
}
97-
}'
93+
if (status != JobExecutionStatus.RUNNING) {
94+
s"""\\nCompletion Time: ${UIUtils.formatDate(new Date(completionTime))}"""
95+
} else {
96+
""
97+
}
98+
}'
9899
|}
99100
""".stripMargin
100-
Some(jobEventJsonAsStr)
101-
}
101+
jobEventJsonAsStr
102102
}
103+
}
103104

104-
def makeExecutorEvent(executorUIDatas: Seq[ExecutorUIData]): Seq[String] = {
105-
val events = ListBuffer[String]()
106-
executorUIDatas.foreach { event =>
105+
private def makeExecutorEvent(executorUIDatas: Seq[ExecutorUIData]): Seq[String] = {
106+
val events = ListBuffer[String]()
107+
executorUIDatas.foreach { event =>
107108

108-
if (event.startTime.isDefined) {
109-
val addedEvent =
110-
s"""
109+
if (event.startTime.isDefined) {
110+
val addedEvent =
111+
s"""
111112
|{
112113
| 'className': 'executor added',
113114
| 'group': 'executors',
@@ -116,12 +117,12 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
116117
| 'title': 'Added at ${UIUtils.formatDate(new Date(event.startTime.get))}'
117118
|}
118119
""".stripMargin
119-
events += addedEvent
120-
}
120+
events += addedEvent
121+
}
121122

122-
if (event.finishTime.isDefined) {
123-
val removedEvent =
124-
s"""
123+
if (event.finishTime.isDefined) {
124+
val removedEvent =
125+
s"""
125126
|{
126127
| 'className': 'executor removed',
127128
| 'group': 'executors',
@@ -137,31 +138,26 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
137138
}'
138139
|}
139140
""".stripMargin
140-
events += removedEvent
141-
}
141+
events += removedEvent
142142
}
143-
events.toSeq
144143
}
144+
events.toSeq
145+
}
146+
147+
private def makeTimeline(
148+
jobs: Seq[JobUIData],
149+
executors: Seq[ExecutorUIData],
150+
startTime: Long): Seq[Node] = {
145151

146152
val jobEventJsonAsStrSeq = makeJobEvent(jobs)
147153
val executorEventJsonAsStrSeq = makeExecutorEvent(executors)
148154

149-
val executorsLegend =
150-
<div class="legend-area"><svg width="200px" height="55px">
151-
<rect x="5px" y="5px" width="20px" height="15px"
152-
rx="2px" ry="2px" stroke="#97B0F8" fill="#D5DDF6"></rect>
153-
<text x="35px" y="17px">Executor Added</text>
154-
<rect x="5px" y="35px" width="20px" height="15px"
155-
rx="2px" ry="2px" stroke="#97B0F8" fill="#EBCA59"></rect>
156-
<text x="35px" y="47px">Executor Removed</text>
157-
</svg></div>.toString.filter(_ != '\n')
158-
159155
val groupJsonArrayAsStr =
160156
s"""
161157
|[
162158
| {
163159
| 'id': 'executors',
164-
| 'content': '<div>Executors</div>${executorsLegend}',
160+
| 'content': '<div>Executors</div>${EXECUTORS_LEGEND}',
165161
| },
166162
| {
167163
| 'id': 'jobs',
@@ -182,7 +178,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
182178
<div id="application-timeline"></div> ++
183179
<script type="text/javascript">
184180
{Unparsed(s"drawApplicationTimeline(${groupJsonArrayAsStr}," +
185-
s"${eventArrayAsStr}, ${startTime.getOrElse(-1L)});")}
181+
s"${eventArrayAsStr}, ${startTime});")}
186182
</script>
187183
}
188184

@@ -201,10 +197,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
201197
def makeRow(job: JobUIData): Seq[Node] = {
202198
val lastStageInfo = Option(job.stageIds)
203199
.filter(_.nonEmpty)
204-
.flatMap { ids => listener.stageIdToInfo.get(ids.max) }
205-
val lastStageData = lastStageInfo.flatMap { s =>
206-
listener.stageIdToData.get((s.stageId, s.attemptId))
207-
}
200+
.flatMap { ids => parent.listener.stageIdToInfo.get(ids.max) }
208201

209202
val lastStageName = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
210203
val lastStageDescription = getlastStageDescription(job)
@@ -252,7 +245,9 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
252245
}
253246

254247
def render(request: HttpServletRequest): Seq[Node] = {
248+
val listener = parent.listener
255249
listener.synchronized {
250+
val startTime = listener.startTime
256251
val activeJobs = listener.activeJobs.values.toSeq
257252
val completedJobs = listener.completedJobs.reverse.toSeq
258253
val failedJobs = listener.failedJobs.reverse.toSeq
@@ -271,11 +266,11 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
271266
val summary: NodeSeq =
272267
<div>
273268
<ul class="unstyled">
274-
{if (startTime.isDefined) {
269+
{if (parent.sc.isDefined) {
275270
// Total duration is not meaningful unless the UI is live
276271
<li>
277272
<strong>Total Duration: </strong>
278-
{UIUtils.formatDuration(System.currentTimeMillis() - startTime.get)}
273+
{UIUtils.formatDuration(System.currentTimeMillis() - startTime)}
279274
</li>
280275
}}
281276
<li>
@@ -310,9 +305,8 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
310305
</div>
311306

312307
var content = summary
313-
val appStartTime =
314308
content ++= <h4>Events on Application Timeline</h4> ++
315-
makeTimeline(activeJobs ++ completedJobs ++ failedJobs, listener.executors)
309+
makeTimeline(activeJobs ++ completedJobs ++ failedJobs, listener.executors, startTime)
316310

317311
if (shouldShowActiveJobs) {
318312
content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++

0 commit comments

Comments
 (0)