|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.spark.ui.jobs |
| 19 | + |
| 20 | +import scala.xml.{Node, NodeSeq} |
| 21 | + |
| 22 | +import javax.servlet.http.HttpServletRequest |
| 23 | + |
| 24 | +import org.apache.spark.JobExecutionStatus |
| 25 | +import org.apache.spark.ui.{WebUIPage, UIUtils} |
| 26 | +import org.apache.spark.ui.jobs.UIData.JobUIData |
| 27 | + |
| 28 | +/** Page showing list of all ongoing and recently finished jobs */ |
| 29 | +private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") { |
| 30 | + private val startTime: Option[Long] = parent.sc.map(_.startTime) |
| 31 | + private val listener = parent.listener |
| 32 | + |
| 33 | + private def jobsTable(jobs: Seq[JobUIData]): Seq[Node] = { |
| 34 | + val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined) |
| 35 | + |
| 36 | + val columns: Seq[Node] = { |
| 37 | + <th>{if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"}</th> |
| 38 | + <th>Description</th> |
| 39 | + <th>Submitted</th> |
| 40 | + <th>Duration</th> |
| 41 | + <th class="sorttable_nosort">Stages: Succeeded/Total</th> |
| 42 | + <th class="sorttable_nosort">Tasks (for all stages): Succeeded/Total</th> |
| 43 | + } |
| 44 | + |
| 45 | + def makeRow(job: JobUIData): Seq[Node] = { |
| 46 | + val lastStageInfo = listener.stageIdToInfo.get(job.stageIds.max) |
| 47 | + val lastStageData = lastStageInfo.flatMap { s => |
| 48 | + listener.stageIdToData.get((s.stageId, s.attemptId)) |
| 49 | + } |
| 50 | + val isComplete = job.status == JobExecutionStatus.SUCCEEDED |
| 51 | + val lastStageName = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)") |
| 52 | + val lastStageDescription = lastStageData.flatMap(_.description).getOrElse("") |
| 53 | + val duration: Option[Long] = { |
| 54 | + job.startTime.map { start => |
| 55 | + val end = job.endTime.getOrElse(System.currentTimeMillis()) |
| 56 | + end - start |
| 57 | + } |
| 58 | + } |
| 59 | + val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown") |
| 60 | + val formattedSubmissionTime = job.startTime.map(UIUtils.formatDate).getOrElse("Unknown") |
| 61 | + val detailUrl = |
| 62 | + "%s/jobs/job?id=%s".format(UIUtils.prependBaseUri(parent.basePath), job.jobId) |
| 63 | + <tr> |
| 64 | + <td sorttable_customkey={job.jobId.toString}> |
| 65 | + {job.jobId} {job.jobGroup.map(id => s"($id)").getOrElse("")} |
| 66 | + </td> |
| 67 | + <td> |
| 68 | + <div><em>{lastStageDescription}</em></div> |
| 69 | + <a href={detailUrl}>{lastStageName}</a> |
| 70 | + </td> |
| 71 | + <td sorttable_customkey={job.startTime.getOrElse(-1).toString}> |
| 72 | + {formattedSubmissionTime} |
| 73 | + </td> |
| 74 | + <td sorttable_customkey={duration.getOrElse(-1).toString}>{formattedDuration}</td> |
| 75 | + <td class="stage-progress-cell"> |
| 76 | + {job.completedStageIndices.size}/{job.stageIds.size - job.numSkippedStages} |
| 77 | + {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"} |
| 78 | + {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"} |
| 79 | + </td> |
| 80 | + <td class="progress-cell"> |
| 81 | + {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks, |
| 82 | + failed = job.numFailedTasks, skipped = job.numSkippedTasks, |
| 83 | + total = job.numTasks - job.numSkippedTasks)} |
| 84 | + </td> |
| 85 | + </tr> |
| 86 | + } |
| 87 | + |
| 88 | + <table class="table table-bordered table-striped table-condensed sortable"> |
| 89 | + <thead>{columns}</thead> |
| 90 | + <tbody> |
| 91 | + {jobs.map(makeRow)} |
| 92 | + </tbody> |
| 93 | + </table> |
| 94 | + } |
| 95 | + |
| 96 | + def render(request: HttpServletRequest): Seq[Node] = { |
| 97 | + listener.synchronized { |
| 98 | + val activeJobs = listener.activeJobs.values.toSeq |
| 99 | + val completedJobs = listener.completedJobs.reverse.toSeq |
| 100 | + val failedJobs = listener.failedJobs.reverse.toSeq |
| 101 | + val now = System.currentTimeMillis |
| 102 | + |
| 103 | + val activeJobsTable = |
| 104 | + jobsTable(activeJobs.sortBy(_.startTime.getOrElse(-1L)).reverse) |
| 105 | + val completedJobsTable = |
| 106 | + jobsTable(completedJobs.sortBy(_.endTime.getOrElse(-1L)).reverse) |
| 107 | + val failedJobsTable = |
| 108 | + jobsTable(failedJobs.sortBy(_.endTime.getOrElse(-1L)).reverse) |
| 109 | + |
| 110 | + val summary: NodeSeq = |
| 111 | + <div> |
| 112 | + <ul class="unstyled"> |
| 113 | + {if (startTime.isDefined) { |
| 114 | + // Total duration is not meaningful unless the UI is live |
| 115 | + <li> |
| 116 | + <strong>Total Duration: </strong> |
| 117 | + {UIUtils.formatDuration(now - startTime.get)} |
| 118 | + </li> |
| 119 | + }} |
| 120 | + <li> |
| 121 | + <strong>Scheduling Mode: </strong> |
| 122 | + {listener.schedulingMode.map(_.toString).getOrElse("Unknown")} |
| 123 | + </li> |
| 124 | + <li> |
| 125 | + <a href="#active"><strong>Active Jobs:</strong></a> |
| 126 | + {activeJobs.size} |
| 127 | + </li> |
| 128 | + <li> |
| 129 | + <a href="#completed"><strong>Completed Jobs:</strong></a> |
| 130 | + {completedJobs.size} |
| 131 | + </li> |
| 132 | + <li> |
| 133 | + <a href="#failed"><strong>Failed Jobs:</strong></a> |
| 134 | + {failedJobs.size} |
| 135 | + </li> |
| 136 | + </ul> |
| 137 | + </div> |
| 138 | + |
| 139 | + val content = summary ++ |
| 140 | + <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++ activeJobsTable ++ |
| 141 | + <h4 id="completed">Completed Jobs ({completedJobs.size})</h4> ++ completedJobsTable ++ |
| 142 | + <h4 id ="failed">Failed Jobs ({failedJobs.size})</h4> ++ failedJobsTable |
| 143 | + |
| 144 | + val helpText = """A job is triggered by a action, like "count()" or "saveAsTextFile()".""" + |
| 145 | + " Click on a job's title to see information about the stages of tasks associated with" + |
| 146 | + " the job." |
| 147 | + |
| 148 | + UIUtils.headerSparkPage("Spark Jobs", content, parent, helpText = Some(helpText)) |
| 149 | + } |
| 150 | + } |
| 151 | +} |
0 commit comments