Skip to content

[SPARK-31638][WEBUI]Clean Pagination code for all webUI pages #28448

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-31638][WEBUI]Clean Pagination code for all webUI pages #28448

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d379a86
Clean pagination code
iRakson May 23, 2020
65007e7
fix
iRakson May 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 5 additions & 18 deletions core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,6 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
val someJobHasJobGroup = jobs.exists(_.jobGroup.isDefined)
val jobIdTitle = if (someJobHasJobGroup) "Job Id (Job Group)" else "Job Id"
val jobPage = Option(request.getParameter(jobTag + ".page")).map(_.toInt).getOrElse(1)
val currentTime = System.currentTimeMillis()

try {
new JobPagedTable(
Expand All @@ -226,7 +225,6 @@ private[ui] class AllJobsPage(parent: JobsTab, store: AppStatusStore) extends We
UIUtils.prependBaseUri(request, parent.basePath),
"jobs", // subPath
killEnabled,
currentTime,
jobIdTitle
).table(jobPage)
} catch {
Expand Down Expand Up @@ -399,7 +397,6 @@ private[ui] class JobDataSource(
store: AppStatusStore,
jobs: Seq[v1.JobData],
basePath: String,
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean) extends PagedDataSource[JobTableRowData](pageSize) {
Expand All @@ -410,15 +407,9 @@ private[ui] class JobDataSource(
// so that we can avoid creating duplicate contents during sorting the data
private val data = jobs.map(jobRow).sorted(ordering(sortColumn, desc))

private var _slicedJobIds: Set[Int] = null

override def dataSize: Int = data.size

override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = {
val r = data.slice(from, to)
_slicedJobIds = r.map(_.jobData.jobId).toSet
r
}
override def sliceData(from: Int, to: Int): Seq[JobTableRowData] = data.slice(from, to)

private def jobRow(jobData: v1.JobData): JobTableRowData = {
val duration: Option[Long] = JobDataUtil.getDuration(jobData)
Expand Down Expand Up @@ -479,17 +470,17 @@ private[ui] class JobPagedTable(
basePath: String,
subPath: String,
killEnabled: Boolean,
currentTime: Long,
jobIdTitle: String
) extends PagedTable[JobTableRowData] {

private val (sortColumn, desc, pageSize) = getTableParameters(request, jobTag, jobIdTitle)
private val parameterPath = basePath + s"/$subPath/?" + getParameterOtherTable(request, jobTag)
private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())

override def tableId: String = jobTag + "-table"

override def tableCssClass: String =
"table table-bordered table-sm table-striped " +
"table-head-clickable table-cell-width-limited"
"table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"

override def pageSizeFormField: String = jobTag + ".pageSize"

Expand All @@ -499,13 +490,11 @@ private[ui] class JobPagedTable(
store,
data,
basePath,
currentTime,
Copy link
Member

@sarutak sarutak May 11, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we can remove currentTime here, can we also remove currentTime from JobPagedTable?

Copy link
Contributor Author

@iRakson iRakson May 11, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah. It can be removed. Thanks.

pageSize,
sortColumn,
desc)

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
parameterPath +
s"&$pageNumberFormField=$page" +
s"&$jobTag.sort=$encodedSortColumn" +
Expand All @@ -514,10 +503,8 @@ private[ui] class JobPagedTable(
s"#$tableHeaderId"
}

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
override def goButtonFormPath: String =
s"$parameterPath&$jobTag.sort=$encodedSortColumn&$jobTag.desc=$desc#$tableHeaderId"
}

override def headers: Seq[Node] = {
// Information for each header: title, sortable, tooltip
Expand Down
14 changes: 3 additions & 11 deletions core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,6 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
stageData,
UIUtils.prependBaseUri(request, parent.basePath) +
s"/stages/stage/?id=${stageId}&attempt=${stageAttemptId}",
currentTime,
pageSize = taskPageSize,
sortColumn = taskSortColumn,
desc = taskSortDesc,
Expand Down Expand Up @@ -452,7 +451,6 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We

private[ui] class TaskDataSource(
stage: StageData,
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean,
Expand All @@ -474,8 +472,6 @@ private[ui] class TaskDataSource(
_tasksToShow
}

def tasks: Seq[TaskData] = _tasksToShow

def executorLogs(id: String): Map[String, String] = {
executorIdToLogs.getOrElseUpdate(id,
store.asOption(store.executorSummary(id)).map(_.executorLogs).getOrElse(Map.empty))
Expand All @@ -486,14 +482,15 @@ private[ui] class TaskDataSource(
private[ui] class TaskPagedTable(
stage: StageData,
basePath: String,
currentTime: Long,
pageSize: Int,
sortColumn: String,
desc: Boolean,
store: AppStatusStore) extends PagedTable[TaskData] {

import ApiHelper._

private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())

override def tableId: String = "task-table"

override def tableCssClass: String =
Expand All @@ -505,25 +502,20 @@ private[ui] class TaskPagedTable(

override val dataSource: TaskDataSource = new TaskDataSource(
stage,
currentTime,
Copy link
Member

@sarutak sarutak May 11, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar to the comment for AllJobsPage, can we remove currentTime from TaskPagedTable?

pageSize,
sortColumn,
desc,
store)

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
basePath +
s"&$pageNumberFormField=$page" +
s"&task.sort=$encodedSortColumn" +
s"&task.desc=$desc" +
s"&$pageSizeFormField=$pageSize"
}

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
s"$basePath&task.sort=$encodedSortColumn&task.desc=$desc"
}
override def goButtonFormPath: String = s"$basePath&task.sort=$encodedSortColumn&task.desc=$desc"

def headers: Seq[Node] = {
import ApiHelper._
Expand Down
21 changes: 6 additions & 15 deletions core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,16 +116,17 @@ private[ui] class StagePagedTable(
override def tableId: String = stageTag + "-table"

override def tableCssClass: String =
"table table-bordered table-sm table-striped " +
"table-head-clickable table-cell-width-limited"
"table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"

override def pageSizeFormField: String = stageTag + ".pageSize"

override def pageNumberFormField: String = stageTag + ".page"

private val (sortColumn, desc, pageSize) = getTableParameters(request, stageTag, "Stage Id")

val parameterPath = UIUtils.prependBaseUri(request, basePath) + s"/$subPath/?" +
private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())

private val parameterPath = UIUtils.prependBaseUri(request, basePath) + s"/$subPath/?" +
getParameterOtherTable(request, stageTag)

override val dataSource = new StageDataSource(
Expand All @@ -138,7 +139,6 @@ private[ui] class StagePagedTable(
)

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
parameterPath +
s"&$pageNumberFormField=$page" +
s"&$stageTag.sort=$encodedSortColumn" +
Expand All @@ -147,10 +147,8 @@ private[ui] class StagePagedTable(
s"#$tableHeaderId"
}

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
override def goButtonFormPath: String =
s"$parameterPath&$stageTag.sort=$encodedSortColumn&$stageTag.desc=$desc#$tableHeaderId"
}

override def headers: Seq[Node] = {
// stageHeadersAndCssClasses has three parts: header title, sortable and tooltip information.
Expand Down Expand Up @@ -311,15 +309,9 @@ private[ui] class StageDataSource(
// table so that we can avoid creating duplicate contents during sorting the data
private val data = stages.map(stageRow).sorted(ordering(sortColumn, desc))

private var _slicedStageIds: Set[Int] = _

override def dataSize: Int = data.size

override def sliceData(from: Int, to: Int): Seq[StageTableRowData] = {
val r = data.slice(from, to)
_slicedStageIds = r.map(_.stageId).toSet
r
}
override def sliceData(from: Int, to: Int): Seq[StageTableRowData] = data.slice(from, to)

private def stageRow(stageData: v1.StageData): StageTableRowData = {
val formattedSubmissionTime = stageData.submissionTime match {
Expand Down Expand Up @@ -350,7 +342,6 @@ private[ui] class StageDataSource(
val shuffleWrite = stageData.shuffleWriteBytes
val shuffleWriteWithUnit = if (shuffleWrite > 0) Utils.bytesToString(shuffleWrite) else ""


new StageTableRowData(
stageData,
Some(stageData),
Expand Down
1 change: 0 additions & 1 deletion core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
val taskTable = new TaskPagedTable(
stageData,
basePath = "/a/b/c",
currentTime = 0,
pageSize = 10,
sortColumn = "Index",
desc = false,
Expand Down
29 changes: 6 additions & 23 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/ui/AllExecutionsPage.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,11 +203,10 @@ private[ui] class ExecutionPagedTable(

private val (sortColumn, desc, pageSize) = getTableParameters(request, executionTag, "ID")

private val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())

override val dataSource = new ExecutionDataSource(
request,
parent,
data,
basePath,
currentTime,
pageSize,
sortColumn,
Expand All @@ -222,11 +221,9 @@ private[ui] class ExecutionPagedTable(
override def tableId: String = s"$executionTag-table"

override def tableCssClass: String =
"table table-bordered table-sm table-striped " +
"table-head-clickable table-cell-width-limited"
"table table-bordered table-sm table-striped table-head-clickable table-cell-width-limited"

override def pageLink(page: Int): String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
parameterPath +
s"&$pageNumberFormField=$page" +
s"&$executionTag.sort=$encodedSortColumn" +
Expand All @@ -239,10 +236,8 @@ private[ui] class ExecutionPagedTable(

override def pageNumberFormField: String = s"$executionTag.page"

override def goButtonFormPath: String = {
val encodedSortColumn = URLEncoder.encode(sortColumn, UTF_8.name())
override def goButtonFormPath: String =
s"$parameterPath&$executionTag.sort=$encodedSortColumn&$executionTag.desc=$desc#$tableHeaderId"
}

override def headers: Seq[Node] = {
// Information for each header: title, sortable, tooltip
Expand Down Expand Up @@ -348,7 +343,6 @@ private[ui] class ExecutionPagedTable(


private[ui] class ExecutionTableRowData(
val submissionTime: Long,
val duration: Long,
val executionUIData: SQLExecutionUIData,
val runningJobData: Seq[Int],
Expand All @@ -357,10 +351,7 @@ private[ui] class ExecutionTableRowData(


private[ui] class ExecutionDataSource(
request: HttpServletRequest,
parent: SQLTab,
executionData: Seq[SQLExecutionUIData],
basePath: String,
currentTime: Long,
pageSize: Int,
sortColumn: String,
Expand All @@ -373,20 +364,13 @@ private[ui] class ExecutionDataSource(
// in the table so that we can avoid creating duplicate contents during sorting the data
private val data = executionData.map(executionRow).sorted(ordering(sortColumn, desc))

private var _sliceExecutionIds: Set[Int] = _

override def dataSize: Int = data.size

override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] = {
val r = data.slice(from, to)
_sliceExecutionIds = r.map(_.executionUIData.executionId.toInt).toSet
r
}
override def sliceData(from: Int, to: Int): Seq[ExecutionTableRowData] = data.slice(from, to)

private def executionRow(executionUIData: SQLExecutionUIData): ExecutionTableRowData = {
val submissionTime = executionUIData.submissionTime
val duration = executionUIData.completionTime.map(_.getTime())
.getOrElse(currentTime) - submissionTime
.getOrElse(currentTime) - executionUIData.submissionTime

val runningJobData = if (showRunningJobs) {
executionUIData.jobs.filter {
Expand All @@ -407,7 +391,6 @@ private[ui] class ExecutionDataSource(
} else Seq.empty

new ExecutionTableRowData(
submissionTime,
duration,
executionUIData,
runningJobData,
Expand Down
Loading