[SPARK-4070] [WIP] Add WebUITableBuilder to simplify table-building code

core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala

@@ -21,12 +21,27 @@ import javax.servlet.http.HttpServletRequest
 
 import scala.xml.Node
 
-import org.apache.spark.ui.{WebUIPage, UIUtils}
+import org.apache.spark.ui.{UITableBuilder, UITable, WebUIPage, UIUtils}
 
 private[spark] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {
 
   private val pageSize = 20
 
+  val appTable: UITable[ApplicationHistoryInfo] = {
+    val t = new UITableBuilder[ApplicationHistoryInfo]()
+    t.col("App ID") (identity) withMarkup { info =>
+      val uiAddress = HistoryServer.UI_PATH_PREFIX + s"/${info.id}"
+      <a href={uiAddress}>{info.id}</a>
+    }
+    t.col("App Name") { _.name }
+    t.epochDateCol("Started") { _.startTime }
+    t.epochDateCol("Completed") { _.endTime }
+    t.durationCol("Duration") { info => info.endTime - info.startTime }
+    t.col("Spark User") { _.sparkUser }
+    t.epochDateCol("Last Updated") { _.lastUpdated }
+    t.build()
+  }
+
   def render(request: HttpServletRequest): Seq[Node] = {
     val requestedPage = Option(request.getParameter("page")).getOrElse("1").toInt
     val requestedFirst = (requestedPage - 1) * pageSize
@@ -39,7 +54,7 @@ private[spark] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {
     val last = Math.min(actualFirst + pageSize, allApps.size) - 1
     val pageCount = allApps.size / pageSize + (if (allApps.size % pageSize > 0) 1 else 0)
 
-    val appTable = UIUtils.listingTable(appHeader, appRow, apps)
+    val appTable = this.appTable.render(apps)
     val providerConfig = parent.getProviderConfig()
     val content =
       <div class="row-fluid">
@@ -65,30 +80,4 @@ private[spark] class HistoryPage(parent: HistoryServer) extends WebUIPage("") {
       </div>
     UIUtils.basicSparkPage(content, "History Server")
   }
-
-  private val appHeader = Seq(
-    "App ID",
-    "App Name",
-    "Started",
-    "Completed",
-    "Duration",
-    "Spark User",
-    "Last Updated")
-
-  private def appRow(info: ApplicationHistoryInfo): Seq[Node] = {
-    val uiAddress = HistoryServer.UI_PATH_PREFIX + s"/${info.id}"
-    val startTime = UIUtils.formatDate(info.startTime)
-    val endTime = UIUtils.formatDate(info.endTime)
-    val duration = UIUtils.formatDuration(info.endTime - info.startTime)
-    val lastUpdated = UIUtils.formatDate(info.lastUpdated)
-    <tr>
-      <td><a href={uiAddress}>{info.id}</a></td>
-      <td>{info.name}</td>
-      <td>{startTime}</td>
-      <td>{endTime}</td>
-      <td>{duration}</td>
-      <td>{info.sparkUser}</td>
-      <td>{lastUpdated}</td>
-    </tr>
-  }
 }

core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala

@@ -28,7 +28,7 @@ import org.json4s.JValue
 import org.apache.spark.deploy.{ExecutorState, JsonProtocol}
 import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
 import org.apache.spark.deploy.master.ExecutorInfo
-import org.apache.spark.ui.{UIUtils, WebUIPage}
+import org.apache.spark.ui.{UITable, UITableBuilder, UIUtils, WebUIPage}
 import org.apache.spark.util.Utils
 
 private[spark] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app") {
@@ -47,6 +47,24 @@ private[spark] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app
     JsonProtocol.writeApplicationInfo(app)
   }
 
+  private val executorsTable: UITable[ExecutorInfo] = {
+    val t = new UITableBuilder[ExecutorInfo]()
+    t.col("ExecutorID") { _.id.toString }
+    t.col("Worker") (identity) withMarkup { executor =>
+      <a href={executor.worker.webUiAddress}>{executor.worker.id}</a>
+    }
+    t.col("Cores") { _.cores }
+    t.sizeCol("Memory") { _.memory }
+    t.col("State") { _.state.toString }
+    t.col("Logs") (identity) withMarkup { executor =>
+      <a href={"%s/logPage?appId=%s&executorId=%s&logType=stdout"
+        .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stdout</a>
+        <a href={"%s/logPage?appId=%s&executorId=%s&logType=stderr"
+          .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a>
+    } isUnsortable()
+    t.build()
+  }
+
   /** Executor details for a particular application */
   def render(request: HttpServletRequest): Seq[Node] = {
     val appId = request.getParameter("appId")
@@ -56,15 +74,14 @@ private[spark] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app
       state.completedApps.find(_.id == appId).getOrElse(null)
     })
 
-    val executorHeaders = Seq("ExecutorID", "Worker", "Cores", "Memory", "State", "Logs")
     val allExecutors = (app.executors.values ++ app.removedExecutors).toSet.toSeq
     // This includes executors that are either still running or have exited cleanly
     val executors = allExecutors.filter { exec =>
       !ExecutorState.isFinished(exec.state) || exec.state == ExecutorState.EXITED
     }
     val removedExecutors = allExecutors.diff(executors)
-    val executorsTable = UIUtils.listingTable(executorHeaders, executorRow, executors)
-    val removedExecutorsTable = UIUtils.listingTable(executorHeaders, executorRow, removedExecutors)
+    val executorsTable = this.executorsTable.render(executors)
+    val removedExecutorsTable = this.executorsTable.render(removedExecutors)
 
     val content =
       <div class="row-fluid">
@@ -108,22 +125,4 @@ private[spark] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app
       </div>;
     UIUtils.basicSparkPage(content, "Application: " + app.desc.name)
   }
-
-  private def executorRow(executor: ExecutorInfo): Seq[Node] = {
-    <tr>
-      <td>{executor.id}</td>
-      <td>
-        <a href={executor.worker.webUiAddress}>{executor.worker.id}</a>
-      </td>
-      <td>{executor.cores}</td>
-      <td>{executor.memory}</td>
-      <td>{executor.state}</td>
-      <td>
-        <a href={"%s/logPage?appId=%s&executorId=%s&logType=stdout"
-          .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stdout</a>
-        <a href={"%s/logPage?appId=%s&executorId=%s&logType=stderr"
-          .format(executor.worker.webUiAddress, executor.application.id, executor.id)}>stderr</a>
-      </td>
-    </tr>
-  }
 }

core/src/main/scala/org/apache/spark/deploy/master/ui/MasterPage.scala

@@ -20,15 +20,15 @@ package org.apache.spark.deploy.master.ui
 import javax.servlet.http.HttpServletRequest
 
 import scala.concurrent.Await
-import scala.xml.Node
+import scala.xml.{Text, Node}
 
 import akka.pattern.ask
 import org.json4s.JValue
 
 import org.apache.spark.deploy.JsonProtocol
 import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
-import org.apache.spark.deploy.master.{ApplicationInfo, DriverInfo, WorkerInfo}
-import org.apache.spark.ui.{WebUIPage, UIUtils}
+import org.apache.spark.deploy.master.{WorkerInfo, ApplicationInfo, DriverInfo}
+import org.apache.spark.ui.{UITable, UITableBuilder, WebUIPage, UIUtils}
 import org.apache.spark.util.Utils
 
 private[spark] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
@@ -41,32 +41,69 @@ private[spark] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
     JsonProtocol.writeMasterState(state)
   }
 
+  private val workerTable: UITable[WorkerInfo] = {
+    val t = new UITableBuilder[WorkerInfo]()
+    t.col("ID") (identity) withMarkup  { worker =>
+      <a href={worker.webUiAddress}>{worker.id}</a>
+    }
+    t.col("Address") { worker => s"${worker.host}:${worker.port}"}
+    t.col("State") { _.state.toString }
+    t.col("Cores") { _.coresUsed } formatWith { c: Int => s"$c Used" }
+    t.col("Memory") (identity) sortBy { worker =>
+      s"${worker.memory}:${worker.memoryUsed}"
+    } withMarkup { worker =>
+      Text(Utils.megabytesToString(worker.memory)) ++
+      Text(Utils.megabytesToString(worker.memoryUsed))
+    }
+    t.build()
+  }
+
+  private val appTable: UITable[ApplicationInfo] = {
+    val t = new UITableBuilder[ApplicationInfo]()
+    t.col("ID") (_.id) withMarkup { id =>
+      <a href={"app?appId=" + id}>{id}</a>
+    }
+    t.col("Name") { _.id }
+    t.col("Cores") { _.coresGranted }
+    t.sizeCol("Memory per Node") { _.desc.memoryPerSlave }
+    t.dateCol("Submitted Time") { _.submitDate }
+    t.col("User") { _.desc.user }
+    t.col("State") { _.state.toString }
+    t.durationCol("Duration") { _.duration }
+    t.build()
+  }
+
+  private val driverTable: UITable[DriverInfo] = {
+    val t = new UITableBuilder[DriverInfo]()
+    t.col("ID") { _.id }
+    t.dateCol("Submitted Time") { _.submitDate }
+    t.col("Worker") (identity) withMarkup { driver =>
+      driver.worker.map(w => <a href={w.webUiAddress}>{w.id.toString}</a>).getOrElse(Text("None"))
+    }
+    t.col("State") { _.state.toString }
+    t.col("Cores") { _.desc.cores }
+    t.sizeCol("Memory") { _.desc.mem.toLong }
+    t.col("Main Class") { _.desc.command.arguments(1) }
+    t.build()
+  }
+
   /** Index view listing applications and executors */
   def render(request: HttpServletRequest): Seq[Node] = {
     val stateFuture = (master ? RequestMasterState)(timeout).mapTo[MasterStateResponse]
     val state = Await.result(stateFuture, timeout)
 
-    val workerHeaders = Seq("Id", "Address", "State", "Cores", "Memory")
-    val workers = state.workers.sortBy(_.id)
-    val workerTable = UIUtils.listingTable(workerHeaders, workerRow, workers)
+    val allWorkersTable = workerTable.render(state.workers.sortBy(_.id))
 
-    val appHeaders = Seq("ID", "Name", "Cores", "Memory per Node", "Submitted Time", "User",
-      "State", "Duration")
-    val activeApps = state.activeApps.sortBy(_.startTime).reverse
-    val activeAppsTable = UIUtils.listingTable(appHeaders, appRow, activeApps)
-    val completedApps = state.completedApps.sortBy(_.endTime).reverse
-    val completedAppsTable = UIUtils.listingTable(appHeaders, appRow, completedApps)
+    val activeAppsTable = appTable.render(state.activeApps.sortBy(_.startTime).reverse)
+    val completedAppsTable = appTable.render(state.completedApps.sortBy(_.endTime).reverse)
 
-    val driverHeaders = Seq("ID", "Submitted Time", "Worker", "State", "Cores", "Memory",
-      "Main Class")
-    val activeDrivers = state.activeDrivers.sortBy(_.startTime).reverse
-    val activeDriversTable = UIUtils.listingTable(driverHeaders, driverRow, activeDrivers)
-    val completedDrivers = state.completedDrivers.sortBy(_.startTime).reverse
-    val completedDriversTable = UIUtils.listingTable(driverHeaders, driverRow, completedDrivers)
+    val activeDriversTable = driverTable.render(state.activeDrivers.sortBy(_.startTime).reverse)
+    val completedDriversTable =
+      driverTable.render(state.completedDrivers.sortBy(_.startTime).reverse)
 
     // For now we only show driver information if the user has submitted drivers to the cluster.
     // This is until we integrate the notion of drivers and applications in the UI.
-    def hasDrivers = activeDrivers.length > 0 || completedDrivers.length > 0
+    def hasDrivers = state.activeDrivers.length > 0 || state.completedDrivers.length > 0
 
     val content =
         <div class="row-fluid">
@@ -93,7 +130,7 @@ private[spark] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
         <div class="row-fluid">
           <div class="span12">
             <h4> Workers </h4>
-            {workerTable}
+            {allWorkersTable}
           </div>
         </div>
 
@@ -138,57 +175,4 @@ private[spark] class MasterPage(parent: MasterWebUI) extends WebUIPage("") {
 
     UIUtils.basicSparkPage(content, "Spark Master at " + state.uri)
   }
-
-  private def workerRow(worker: WorkerInfo): Seq[Node] = {
-    <tr>
-      <td>
-        <a href={worker.webUiAddress}>{worker.id}</a>
-      </td>
-      <td>{worker.host}:{worker.port}</td>
-      <td>{worker.state}</td>
-      <td>{worker.cores} ({worker.coresUsed} Used)</td>
-      <td sorttable_customkey={"%s.%s".format(worker.memory, worker.memoryUsed)}>
-        {Utils.megabytesToString(worker.memory)}
-        ({Utils.megabytesToString(worker.memoryUsed)} Used)
-      </td>
-    </tr>
-  }
-
-  private def appRow(app: ApplicationInfo): Seq[Node] = {
-    <tr>
-      <td>
-        <a href={"app?appId=" + app.id}>{app.id}</a>
-      </td>
-      <td>
-        <a href={app.desc.appUiUrl}>{app.desc.name}</a>
-      </td>
-      <td>
-        {app.coresGranted}
-      </td>
-      <td sorttable_customkey={app.desc.memoryPerSlave.toString}>
-        {Utils.megabytesToString(app.desc.memoryPerSlave)}
-      </td>
-      <td>{UIUtils.formatDate(app.submitDate)}</td>
-      <td>{app.desc.user}</td>
-      <td>{app.state.toString}</td>
-      <td>{UIUtils.formatDuration(app.duration)}</td>
-    </tr>
-  }
-
-  private def driverRow(driver: DriverInfo): Seq[Node] = {
-    <tr>
-      <td>{driver.id} </td>
-      <td>{driver.submitDate}</td>
-      <td>{driver.worker.map(w => <a href={w.webUiAddress}>{w.id.toString}</a>).getOrElse("None")}
-      </td>
-      <td>{driver.state}</td>
-      <td sorttable_customkey={driver.desc.cores.toString}>
-        {driver.desc.cores}
-      </td>
-      <td sorttable_customkey={driver.desc.mem.toString}>
-        {Utils.megabytesToString(driver.desc.mem.toLong)}
-      </td>
-      <td>{driver.desc.command.arguments(1)}</td>
-    </tr>
-  }
 }