From f66f2eabfbd98b01ba21bb7bc718507963086978 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 7 Jun 2016 14:31:35 -0500 Subject: [PATCH 01/20] Add AllExecutors REST Endpoint --- .../api/v1/AllExecutorListResource.scala | 43 +++++++++++++++++++ .../spark/status/api/v1/ApiRootResource.scala | 16 +++++++ docs/monitoring.md | 6 ++- 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala new file mode 100644 index 000000000000..0f5fd8f425d2 --- /dev/null +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala @@ -0,0 +1,43 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.spark.status.api.v1 + +import javax.ws.rs.{GET, PathParam, Produces} +import javax.ws.rs.core.MediaType + +import org.apache.spark.ui.SparkUI +import org.apache.spark.ui.exec.ExecutorsPage + +@Produces(Array(MediaType.APPLICATION_JSON)) +private[v1] class AllExecutorListResource(ui: SparkUI) { + + @GET + def executorList(): Seq[ExecutorSummary] = { + val listener = ui.executorsListener + listener.synchronized { + // The follow codes should be protected by `listener` to make sure no executors will be + // removed before we query their status. See SPARK-12784. + val storageStatusList = listener.activeStorageStatusList + val deadStorageStatusList = listener.deadStorageStatusList + (0 until storageStatusList.size).map { statusId => + ExecutorsPage.getExecInfo(listener, statusId, isActive = true) + } ++ (0 until deadStorageStatusList.size).map { statusId => + ExecutorsPage.getExecInfo(listener, statusId, isActive = false) + } + } + } +} diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala index 681f295006e3..de927117e1f6 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala @@ -91,6 +91,13 @@ private[v1] class ApiRootResource extends UIRootFromServletContext { } } + @Path("applications/{appId}/allexecutors") + def getAllExecutors(@PathParam("appId") appId: String): AllExecutorListResource = { + uiRoot.withSparkUI(appId, None) { ui => + new AllExecutorListResource(ui) + } + } + @Path("applications/{appId}/{attemptId}/executors") def getExecutors( @PathParam("appId") appId: String, @@ -100,6 +107,15 @@ private[v1] class ApiRootResource extends UIRootFromServletContext { } } + @Path("applications/{appId}/{attemptId}/allexecutors") + def getAllExecutors( + @PathParam("appId") appId: String, + @PathParam("attemptId") attemptId: String): AllExecutorListResource = { + uiRoot.withSparkUI(appId, Some(attemptId)) { ui => + new AllExecutorListResource(ui) + } + } + @Path("applications/{appId}/stages") def getStages(@PathParam("appId") appId: String): AllStagesResource = { diff --git a/docs/monitoring.md b/docs/monitoring.md index fa6c899a40b6..cae4a49c93c2 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -288,7 +288,11 @@ where `[base-app-id]` is the YARN application ID. /applications/[app-id]/executors - A list of all executors for the given application. + A list of all active executors for the given application. + + + /applications/[app-id]/executors + A list of all(active/dead) executors for the given application. /applications/[app-id]/storage/rdd From 8df49aff99324ebae231f94284d1956617efbf72 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 7 Jun 2016 21:26:48 -0500 Subject: [PATCH 02/20] Adding ExecutorsPage template --- .../ui/static/executorspage-template.html | 102 ++++ .../apache/spark/ui/static/executorspage.js | 437 ++++++++++++++++++ .../scala/org/apache/spark/ui/UIUtils.scala | 4 +- .../apache/spark/ui/exec/ExecutorsPage.scala | 77 +-- 4 files changed, 549 insertions(+), 71 deletions(-) create mode 100644 core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html create mode 100644 core/src/main/resources/org/apache/spark/ui/static/executorspage.js diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html new file mode 100644 index 000000000000..021d1b22d6b6 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html @@ -0,0 +1,102 @@ + + + diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js new file mode 100644 index 000000000000..4e170def1304 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// this function works exactly the same as UIUtils.formatDuration +function formatDuration(milliseconds, type) { + if(type !== 'display') return milliseconds; + if (milliseconds < 100) { + return milliseconds + " ms"; + } + var seconds = milliseconds * 1.0 / 1000; + if (seconds < 1) { + return seconds.toFixed(1) + " s"; + } + if (seconds < 60) { + return seconds.toFixed(0) + " s"; + } + var minutes = seconds / 60; + if (minutes < 10) { + return minutes.toFixed(1) + " min"; + } else if (minutes < 60) { + return minutes.toFixed(0) + " min"; + } + var hours = minutes / 60; + return hours.toFixed(1) + " h"; +} + +function makeIdNumeric(id) { + var strs = id.split("_"); + if (strs.length < 3) { + return id; + } + var appSeqNum = strs[2]; + var resl = strs[0] + "_" + strs[1] + "_"; + var diff = 10 - appSeqNum.length; + while (diff > 0) { + resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters + diff--; + } + resl += appSeqNum; + return resl; +} + +function formatDate(date) { + return date.split(".")[0].replace("T", " "); +} + +function getParameterByName(name, searchString) { + var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), + results = regex.exec(searchString); + return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); +} + +function formatStatus(status, type) { + if(type !== 'display') return status; + if(status) { + return "Active" + } else { + return "Dead" + } +} + +function formatBytes(bytes,type) { + if(type !== 'display') return bytes; + if(bytes == 0) return '0 B'; + var k = 1000; + var dm = 3; + var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; +} + + +jQuery.extend( jQuery.fn.dataTableExt.oSort, { + "title-numeric-pre": function ( a ) { + var x = a.match(/title="*(-?[0-9\.]+)/)[1]; + return parseFloat( x ); + }, + + "title-numeric-asc": function ( a, b ) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "title-numeric-desc": function ( a, b ) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +} ); + +jQuery.extend( jQuery.fn.dataTableExt.oSort, { + "appid-numeric-pre": function ( a ) { + var x = a.match(/title="*(-?[0-9a-zA-Z\-\_]+)/)[1]; + return makeIdNumeric(x); + }, + + "appid-numeric-asc": function ( a, b ) { + return ((a < b) ? -1 : ((a > b) ? 1 : 0)); + }, + + "appid-numeric-desc": function ( a, b ) { + return ((a < b) ? 1 : ((a > b) ? -1 : 0)); + } +} ); + +$(document).ajaxStop($.unblockUI); +$(document).ajaxStart(function(){ + $.blockUI({ message: '

Loading Executors Page...

'}); +}); + +function createBaseURI() { + var parser = document.createElement('a'); + var words = parser.baseURI.split('/'); + var ind = words.indexOf("proxy"); + var appId = words[ind + 1]; + var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId; + return baseURI; +} + +function createRESTEndPoint() { + var parser = document.createElement('a'); + var words = parser.baseURI.split('/'); + var ind = words.indexOf("proxy"); + var appId = words[ind + 1]; + var newBaseURI = words.slice(0, ind + 2).join('/'); + + return newBaseURI + "/api/v1/applications/" + appId +"/allexecutors" + +} + +function formatLogsCells(execLogs, type) { + if(type !== 'display') return Object.keys(execLogs); + if(!execLogs) return; + var result = ''; + $.each(execLogs, function(logName, logUrl) { + result += '
' + logName + '
' + }); + return result; +} + +// Determine Color Opacity from 0.5-1 +// activeTasks range from 0 to maxTasks +function activeTasksAlpha(activeTasks, maxTasks) { + return maxTasks > 0 ? ((activeTasks / maxTasks) * 0.5 + 0.5) : 1; +} + +function activeTasksStyle(activeTasks, maxTasks) { + return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; +} + +function activeTasksColor(activeTasks, maxTasks) { + return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; +} + +// failedTasks range max at 10% failure, alpha max = 1 +function failedTasksAlpha(failedTasks, totalTasks) { + return totalTasks > 0 ? + (Math.min(10 * failedTasks / totalTasks, 1) * 0.5 + 0.5) : 1; +} + +function failedTasksStyle(failedTasks, totalTasks){ + return failedTasks > 0 ? + ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; +} + +function failedTasksColor(failedTasks, totalTasks){ + return failedTasks > 0 ? "white" : "black"; +} + +// totalDuration range from 0 to 50% GC time, alpha max = 1 +function totalDurationAlpha(totalGCTime, totalDuration) { + return totalDuration > 0 ? + (Math.min(totalGCTime / totalDuration + 0.5, 1)) : 1; +} + +function totalDurationStyle(totalGCTime, totalDuration) { + // Red if GC time over GCTimePercent of total time + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + var GCTimePercent = 0.1; + return (totalGCTime > GCTimePercent * totalDuration) ? + ("hsla(0, 100%, 50%, " + totalDurationAlpha(totalGCTime, totalDuration) + ")") : ""; +} + +function totalDurationColor(totalGCTime, totalDuration) { + // Red if GC time over GCTimePercent of total time + // When GCTimePercent is edited change ToolTips.TASK_TIME to match + var GCTimePercent = 0.1; + return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black"; +} + +$(document).ready(function() { + $.extend( $.fn.dataTable.defaults, { + stateSave: true, + lengthMenu: [[20,40,60,100,-1], [20, 40, 60, 100, "All"]], + pageLength: 20 + }); + + executorsSummary = $("#active-executors"); + searchString = executorsSummary["context"]["location"]["search"]; + requestedIncomplete = getParameterByName("showIncomplete", searchString); + requestedIncomplete = (requestedIncomplete == "true" ? true : false); + + var endPoint = createRESTEndPoint(); + + $.getJSON(endPoint, function(response,status,jqXHR) { + var summary=[]; + var allExecCnt = 0; + var allRDDBlocks = 0; + var allMemoryUsed = 0; + var allMaxMemory = 0; + var allDiskUsed = 0; + var allTotalCores = 0; + var allMaxTasks = 0; + var allActiveTasks = 0; + var allFailedTasks = 0; + var allCompletedTasks = 0; + var allTotalTasks = 0; + var allTotalDuration = 0; + var allTotalGCTime = 0; + var allTotalInputBytes = 0; + var allTotalShuffleRead = 0; + var allTotalShuffleWrite = 0; + + var activeExecCnt = 0; + var activeRDDBlocks = 0; + var activeMemoryUsed = 0; + var activeMaxMemory = 0; + var activeDiskUsed = 0; + var activeTotalCores = 0; + var activeMaxTasks = 0; + var activeActiveTasks = 0; + var activeFailedTasks = 0; + var activeCompletedTasks = 0; + var activeTotalTasks = 0; + var activeTotalDuration = 0; + var activeTotalGCTime = 0; + var activeTotalInputBytes = 0; + var activeTotalShuffleRead = 0; + var activeTotalShuffleWrite = 0; + + var deadExecCnt = 0; + var deadRDDBlocks = 0; + var deadMemoryUsed = 0; + var deadMaxMemory = 0; + var deadDiskUsed = 0; + var deadTotalCores = 0; + var deadMaxTasks = 0; + var deadActiveTasks = 0; + var deadFailedTasks = 0; + var deadCompletedTasks = 0; + var deadTotalTasks = 0; + var deadTotalDuration = 0; + var deadTotalGCTime = 0; + var deadTotalInputBytes = 0; + var deadTotalShuffleRead = 0; + var deadTotalShuffleWrite = 0; + + response.forEach(function (exec) { + allExecCnt +=1; + allRDDBlocks += exec.rddBlocks; + allMemoryUsed += exec.memoryUsed; + allMaxMemory += exec.maxMemory; + allDiskUsed += exec.diskUsed; + allTotalCores += exec.totalCores; + allMaxTasks += exec.maxTasks; + allActiveTasks += exec.activeTasks; + allFailedTasks += exec.failedTasks; + allCompletedTasks += exec.completedTasks; + allTotalTasks += exec.totalTasks; + allTotalDuration += exec.totalDuration; + allTotalGCTime += exec.totalGCTime; + allTotalInputBytes += exec.totalInputBytes; + allTotalShuffleRead += exec.totalShuffleRead; + allTotalShuffleWrite += exec.totalShuffleWrite; + if(exec.isActive) { + activeExecCnt +=1; + activeRDDBlocks += exec.rddBlocks; + activeMemoryUsed += exec.memoryUsed; + activeMaxMemory += exec.maxMemory; + activeDiskUsed += exec.diskUsed; + activeTotalCores += exec.totalCores; + activeMaxTasks += exec.maxTasks; + activeActiveTasks += exec.activeTasks; + activeFailedTasks += exec.failedTasks; + activeCompletedTasks += exec.completedTasks; + activeTotalTasks += exec.totalTasks; + activeTotalDuration += exec.totalDuration; + activeTotalGCTime += exec.totalGCTime; + activeTotalInputBytes += exec.totalInputBytes; + activeTotalShuffleRead += exec.totalShuffleRead; + activeTotalShuffleWrite += exec.totalShuffleWrite; + } else { + deadExecCnt +=1; + deadRDDBlocks += exec.rddBlocks; + deadMemoryUsed += exec.memoryUsed; + deadMaxMemory += exec.maxMemory; + deadDiskUsed += exec.diskUsed; + deadTotalCores += exec.totalCores; + deadMaxTasks += exec.maxTasks; + deadActiveTasks += exec.activeTasks; + deadFailedTasks += exec.failedTasks; + deadCompletedTasks += exec.completedTasks; + deadTotalTasks += exec.totalTasks; + deadTotalDuration += exec.totalDuration; + deadTotalGCTime += exec.totalGCTime; + deadTotalInputBytes += exec.totalInputBytes; + deadTotalShuffleRead += exec.totalShuffleRead; + deadTotalShuffleWrite += exec.totalShuffleWrite; + } + }); + + var totalSummary = { "execCnt" : ( "Total(" + allExecCnt + ")"), "allRDDBlocks" : allRDDBlocks, "allMemoryUsed" : allMemoryUsed, + "allMaxMemory" : allMaxMemory, "allDiskUsed" : allDiskUsed, "allTotalCores" : allTotalCores, "allMaxTasks" : allMaxTasks, + "allActiveTasks" : allActiveTasks, "allFailedTasks" : allFailedTasks, "allCompletedTasks" : + allCompletedTasks, "allTotalTasks" : allTotalTasks, "allTotalDuration" : allTotalDuration, + "allTotalGCTime" : allTotalGCTime, "allTotalInputBytes" : allTotalInputBytes, + "allTotalShuffleRead" : allTotalShuffleRead, "allTotalShuffleWrite" : allTotalShuffleWrite }; + var activeSummary = { "execCnt" : ( "Active(" + activeExecCnt + ")"), "allRDDBlocks" : activeRDDBlocks, "allMemoryUsed" : activeMemoryUsed, + "allMaxMemory" : activeMaxMemory, "allDiskUsed" : activeDiskUsed, "allTotalCores" : activeTotalCores, "allMaxTasks" : activeMaxTasks, + "allActiveTasks" : activeActiveTasks, "allFailedTasks" : activeFailedTasks, "allCompletedTasks" : + activeCompletedTasks, "allTotalTasks" : activeTotalTasks, "allTotalDuration" : activeTotalDuration, + "allTotalGCTime" : activeTotalGCTime, "allTotalInputBytes" : activeTotalInputBytes, + "allTotalShuffleRead" : activeTotalShuffleRead, "allTotalShuffleWrite" : activeTotalShuffleWrite }; + var deadSummary = { "execCnt" : ( "Dead(" + deadExecCnt + ")" ), "allRDDBlocks" : deadRDDBlocks, "allMemoryUsed" : deadMemoryUsed, + "allMaxMemory" : deadMaxMemory, "allDiskUsed" : deadDiskUsed, "allTotalCores" : deadTotalCores, "allMaxTasks" : deadMaxTasks, + "allActiveTasks" : deadActiveTasks, "allFailedTasks" : deadFailedTasks, "allCompletedTasks" : + deadCompletedTasks, "allTotalTasks" : deadTotalTasks, "allTotalDuration" : deadTotalDuration, + "allTotalGCTime" : deadTotalGCTime, "allTotalInputBytes" : deadTotalInputBytes, + "allTotalShuffleRead" : deadTotalShuffleRead, "allTotalShuffleWrite" : deadTotalShuffleWrite }; + + var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; + $.get(createBaseURI() + "/static/executorspage-template.html", function(template) { + + executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(),data)); + var selector = "#active-executors-table"; + var conf = { + "data": response, + "columns": [ + //{data: 'id'}, + {data: function(row, type) { return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id;} }, + {data: 'hostPort'}, + {data: 'isActive', render: formatStatus }, + {data: 'rddBlocks'}, + {data: function(row, type) { return type === 'display' ? (formatBytes(row.memoryUsed, type) + ' / ' + formatBytes(row.maxMemory, type)) : row.memoryUsed; } }, + {data: 'diskUsed', render: formatBytes }, + {data: 'totalCores'}, + {data: 'activeTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( sData > 0 ) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); + } } }, + {data: 'failedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( sData > 0 ) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); + } } }, + {data: 'completedTasks'}, + {data: 'totalTasks'}, + {data: function(row, type) { return type === 'display' ? (formatDuration(row.totalDuration, type) + ' (' + formatDuration(row.totalGCTime, type) + ')') : row.totalDuration }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( oData.totalDuration > 0 ) { + $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); + $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); + } } }, + {data: 'totalInputBytes', render: formatBytes }, + {data: 'totalShuffleRead', render: formatBytes }, + {data: 'totalShuffleWrite', render: formatBytes }, + {data: 'executorLogs', render: formatLogsCells }, + {data: 'id', render: function(data, type) {return type === 'display' ? ("Thread Dump" ) : data; }} + ], + "order": [[ 0, "asc" ]] + }; + + $(selector).DataTable(conf); + $('#active-executors [data-toggle="tooltip"]').tooltip(); + + var sumSelector = "#summary-execs-table"; + var sumConf = { + "data": [activeSummary, deadSummary, totalSummary], + "columns": [ + {data: 'execCnt', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + $(nTd).css('font-weight', 'bold'); + } }, + {data: 'allRDDBlocks'}, + {data: function(row, type) { return type === 'display' ? (formatBytes(row.allMemoryUsed, type) + ' / ' + formatBytes(row.allMaxMemory, type)): row.allMemoryUsed; } }, + {data: 'allDiskUsed', render: formatBytes}, + {data: 'allTotalCores'}, + {data: 'allActiveTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( sData > 0 ) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); + } } }, + {data: 'allFailedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( sData > 0 ) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); + } } }, + {data: 'allCompletedTasks'}, + {data: 'allTotalTasks'}, + {data: function(row, type) { return type === 'display' ? (formatDuration(row.allTotalDuration, type) + ' (' + formatDuration(row.allTotalGCTime, type) + ')') : row.allTotalDuration }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if ( oData.allTotalDuration > 0 ) { + $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); + $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); + } } }, + {data: 'allTotalInputBytes', render: formatBytes }, + {data: 'allTotalShuffleRead', render: formatBytes }, + {data: 'allTotalShuffleWrite', render: formatBytes } + ], + "paging": false, + "searching": false, + "info": false + + }; + + $(sumSelector).DataTable(sumConf); + $('#execSummary [data-toggle="tooltip"]').tooltip(); + + }); + }); +}); diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 4e2fe5e0e5be..bea8020f461d 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -200,7 +200,8 @@ private[spark] object UIUtils extends Logging { activeTab: SparkUITab, refreshInterval: Option[Int] = None, helpText: Option[String] = None, - showVisualization: Boolean = false): Seq[Node] = { + showVisualization: Boolean = false, + useDataTables: Boolean = false): Seq[Node] = { val appName = activeTab.appName val shortAppName = if (appName.length < 36) appName else appName.take(32) + "..." @@ -215,6 +216,7 @@ private[spark] object UIUtils extends Logging { {commonHeaderNodes} {if (showVisualization) vizHeaderNodes else Seq.empty} + {if (useDataTables) dataTablesHeaderNodes else Seq.empty} {appName} - {title} diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 67deb7b14bcb..091894e787c3 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -20,7 +20,6 @@ package org.apache.spark.ui.exec import java.net.URLEncoder import javax.servlet.http.HttpServletRequest -import scala.util.Try import scala.xml.Node import org.apache.spark.status.api.v1.ExecutorSummary @@ -54,78 +53,16 @@ private[ui] class ExecutorsPage( // When GCTimePercent is edited change ToolTips.TASK_TIME to match private val GCTimePercent = 0.1 - // a safe String to Int for sorting ids (converts non-numeric Strings to -1) - private def idStrToInt(str: String) : Int = Try(str.toInt).getOrElse(-1) - def render(request: HttpServletRequest): Seq[Node] = { - val (activeExecutorInfo, deadExecutorInfo) = listener.synchronized { - // The follow codes should be protected by `listener` to make sure no executors will be - // removed before we query their status. See SPARK-12784. - val _activeExecutorInfo = { - for (statusId <- 0 until listener.activeStorageStatusList.size) - yield ExecutorsPage.getExecInfo(listener, statusId, isActive = true) - } - val _deadExecutorInfo = { - for (statusId <- 0 until listener.deadStorageStatusList.size) - yield ExecutorsPage.getExecInfo(listener, statusId, isActive = false) - } - (_activeExecutorInfo, _deadExecutorInfo) - } - - val execInfo = activeExecutorInfo ++ deadExecutorInfo - implicit val idOrder = Ordering[Int].on((s: String) => idStrToInt(s)).reverse - val execInfoSorted = execInfo.sortBy(_.id) - val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty - - val execTable = { - - - - - - - - - - - - - - - - - - {if (logsExist) else Seq.empty} - {if (threadDumpEnabled) else Seq.empty} - - - {execInfoSorted.map(execRow(_, logsExist))} - -
Executor IDAddressStatusRDD BlocksStorage MemoryDisk UsedCoresActive TasksFailed TasksComplete TasksTotal TasksTask Time (GC Time)InputShuffle Read - - - Shuffle Write - - LogsThread Dump
- } - val content = -
-
-

Summary

- {execSummary(activeExecutorInfo, deadExecutorInfo)} -
-
-
-
-

Executors

- {execTable} -
+
+ { +
++ + + }
; - UIUtils.headerSparkPage("Executors", content, parent) + UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true) } /** Render an HTML row representing an executor */ @@ -141,7 +78,7 @@ private[ui] class ExecutorsPage( } - {info.id} + {info.id} {info.hostPort} {executorStatus} From c0e2afabd3e05121ab24f7726c3a2e60371ea0ca Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 14 Jun 2016 10:04:34 -0500 Subject: [PATCH 03/20] History page for Executors use mustache template --- .../apache/spark/ui/static/executorspage.js | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 4e170def1304..4d320dea6b5a 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -119,24 +119,40 @@ $(document).ajaxStart(function(){ $.blockUI({ message: '

Loading Executors Page...

'}); }); -function createBaseURI() { +function createTemplateURI() { var parser = document.createElement('a'); var words = parser.baseURI.split('/'); var ind = words.indexOf("proxy"); - var appId = words[ind + 1]; - var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId; - return baseURI; + if(ind > 0) { + var appId = words[ind + 1]; + var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/executorspage-template.html'; + return baseURI; + } else { + ind = words.indexOf("history"); + var baseURI = words.slice(0, ind).join('/') + '/static/executorspage-template.html'; + return baseURI; + } } function createRESTEndPoint() { var parser = document.createElement('a'); var words = parser.baseURI.split('/'); var ind = words.indexOf("proxy"); - var appId = words[ind + 1]; - var newBaseURI = words.slice(0, ind + 2).join('/'); - - return newBaseURI + "/api/v1/applications/" + appId +"/allexecutors" - + if(ind > 0) { + var appId = words[ind + 1]; + var newBaseURI = words.slice(0, ind + 2).join('/'); + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors" + } else { + ind = words.indexOf("history"); + var appId = words[ind + 1]; + var attemptId = words[ind + 2]; + var newBaseURI = words.slice(0, ind).join('/'); + if(isNaN(attemptId) ) { + return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors"; + } else { + return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors"; + } + } } function formatLogsCells(execLogs, type) { @@ -340,7 +356,7 @@ $(document).ready(function() { "allTotalShuffleRead" : deadTotalShuffleRead, "allTotalShuffleWrite" : deadTotalShuffleWrite }; var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; - $.get(createBaseURI() + "/static/executorspage-template.html", function(template) { + $.get(createTemplateURI(), function(template) { executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(),data)); var selector = "#active-executors-table"; From c73e905c092adfaf3faa5cb3f09d4d5a7cc071f9 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Mon, 27 Jun 2016 15:25:47 -0500 Subject: [PATCH 04/20] Remove unused javascript methods --- .../apache/spark/ui/static/executorspage.js | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 4d320dea6b5a..9f08da833323 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -38,32 +38,6 @@ function formatDuration(milliseconds, type) { return hours.toFixed(1) + " h"; } -function makeIdNumeric(id) { - var strs = id.split("_"); - if (strs.length < 3) { - return id; - } - var appSeqNum = strs[2]; - var resl = strs[0] + "_" + strs[1] + "_"; - var diff = 10 - appSeqNum.length; - while (diff > 0) { - resl += "0"; // padding 0 before the app sequence number to make sure it has 10 characters - diff--; - } - resl += appSeqNum; - return resl; -} - -function formatDate(date) { - return date.split(".")[0].replace("T", " "); -} - -function getParameterByName(name, searchString) { - var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"), - results = regex.exec(searchString); - return results === null ? "" : decodeURIComponent(results[1].replace(/\+/g, " ")); -} - function formatStatus(status, type) { if(type !== 'display') return status; if(status) { @@ -99,21 +73,6 @@ jQuery.extend( jQuery.fn.dataTableExt.oSort, { } } ); -jQuery.extend( jQuery.fn.dataTableExt.oSort, { - "appid-numeric-pre": function ( a ) { - var x = a.match(/title="*(-?[0-9a-zA-Z\-\_]+)/)[1]; - return makeIdNumeric(x); - }, - - "appid-numeric-asc": function ( a, b ) { - return ((a < b) ? -1 : ((a > b) ? 1 : 0)); - }, - - "appid-numeric-desc": function ( a, b ) { - return ((a < b) ? 1 : ((a > b) ? -1 : 0)); - } -} ); - $(document).ajaxStop($.unblockUI); $(document).ajaxStart(function(){ $.blockUI({ message: '

Loading Executors Page...

'}); @@ -224,8 +183,6 @@ $(document).ready(function() { executorsSummary = $("#active-executors"); searchString = executorsSummary["context"]["location"]["search"]; - requestedIncomplete = getParameterByName("showIncomplete", searchString); - requestedIncomplete = (requestedIncomplete == "true" ? true : false); var endPoint = createRESTEndPoint(); From 2453e21d9ef415725c13855555876abce4b20344 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 12 Jul 2016 21:45:32 +0000 Subject: [PATCH 05/20] Fix the tooltip placement for detailed executors table headers --- .../ui/static/executorspage-template.html | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html index 021d1b22d6b6..2039bf56035b 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html @@ -60,27 +60,27 @@

Executors

Executor ID - Address - Status + Address + Status - RDD Blocks + RDD Blocks - Storage Memory - Disk Used - Cores - Active Tasks - Failed Tasks - Complete Tasks - Total Tasks - Disk Used + Cores + Active Tasks + Failed Tasks + Complete Tasks + Total Tasks + Task Time (GC Time) - Input - Shuffle Read From 55419aad1e4db20cbed136c97a51d11a97fff320 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 12 Jul 2016 22:24:54 +0000 Subject: [PATCH 06/20] Fix the baseURI reference for safari --- .../resources/org/apache/spark/ui/static/executorspage.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 9f08da833323..ad4347179555 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -80,7 +80,7 @@ $(document).ajaxStart(function(){ function createTemplateURI() { var parser = document.createElement('a'); - var words = parser.baseURI.split('/'); + var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); if(ind > 0) { var appId = words[ind + 1]; @@ -95,7 +95,7 @@ function createTemplateURI() { function createRESTEndPoint() { var parser = document.createElement('a'); - var words = parser.baseURI.split('/'); + var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); if(ind > 0) { var appId = words[ind + 1]; From fc3dbd74ff7309ac13d0123876e005e869144714 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 12 Jul 2016 22:41:23 +0000 Subject: [PATCH 07/20] Removed unused imports and refactor to use in-line references --- .../spark/status/api/v1/AllExecutorListResource.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala index 0f5fd8f425d2..01f2a18122e6 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllExecutorListResource.scala @@ -16,7 +16,7 @@ */ package org.apache.spark.status.api.v1 -import javax.ws.rs.{GET, PathParam, Produces} +import javax.ws.rs.{GET, Produces} import javax.ws.rs.core.MediaType import org.apache.spark.ui.SparkUI @@ -31,11 +31,9 @@ private[v1] class AllExecutorListResource(ui: SparkUI) { listener.synchronized { // The follow codes should be protected by `listener` to make sure no executors will be // removed before we query their status. See SPARK-12784. - val storageStatusList = listener.activeStorageStatusList - val deadStorageStatusList = listener.deadStorageStatusList - (0 until storageStatusList.size).map { statusId => + (0 until listener.activeStorageStatusList.size).map { statusId => ExecutorsPage.getExecInfo(listener, statusId, isActive = true) - } ++ (0 until deadStorageStatusList.size).map { statusId => + } ++ (0 until listener.deadStorageStatusList.size).map { statusId => ExecutorsPage.getExecInfo(listener, statusId, isActive = false) } } From 345419398cf82c1e9b1360a622b9b874154677e7 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Tue, 12 Jul 2016 22:44:46 +0000 Subject: [PATCH 08/20] Lower heading level for summary to 4 --- .../org/apache/spark/ui/static/executorspage-template.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html index 2039bf56035b..2bdb07a3fab2 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html @@ -16,7 +16,7 @@ --> ++ }
; From eb96c5d89077041418c3c667e579373a9e1ddedb Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 13 Jul 2016 13:22:48 -0500 Subject: [PATCH 11/20] Refactor to use common methods from utils.js --- .../org/apache/spark/ui/static/historypage.js | 22 -- .../spark/deploy/history/HistoryPage.scala | 5 +- .../apache/spark/ui/exec/ExecutorsPage.scala | 238 ++++++++++-------- 3 files changed, 130 insertions(+), 135 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index d2161662d567..5b9afb59ef8e 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -15,28 +15,6 @@ * limitations under the License. */ -// this function works exactly the same as UIUtils.formatDuration -function formatDuration(milliseconds) { - if (milliseconds < 100) { - return milliseconds + " ms"; - } - var seconds = milliseconds * 1.0 / 1000; - if (seconds < 1) { - return seconds.toFixed(1) + " s"; - } - if (seconds < 60) { - return seconds.toFixed(0) + " s"; - } - var minutes = seconds / 60; - if (minutes < 10) { - return minutes.toFixed(1) + " min"; - } else if (minutes < 60) { - return minutes.toFixed(0) + " min"; - } - var hours = minutes / 60; - return hours.toFixed(1) + " h"; -} - function makeIdNumeric(id) { var strs = id.split("_"); if (strs.length < 3) { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala index 2fad1120cdc8..74f78021ed6c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala @@ -43,8 +43,9 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("") { if (allAppsSize > 0) { ++ - ++ - + ++ + ++ + } else if (requestedIncomplete) {

No incomplete applications found!

} else { diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index f0da0910fdca..80a3ab260e6b 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -28,26 +28,26 @@ import org.apache.spark.util.Utils // This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive private[ui] case class ExecutorSummaryInfo( - id: String, - hostPort: String, - rddBlocks: Int, - memoryUsed: Long, - diskUsed: Long, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalInputBytes: Long, - totalShuffleRead: Long, - totalShuffleWrite: Long, - maxMemory: Long, - executorLogs: Map[String, String]) + id: String, + hostPort: String, + rddBlocks: Int, + memoryUsed: Long, + diskUsed: Long, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalInputBytes: Long, + totalShuffleRead: Long, + totalShuffleWrite: Long, + maxMemory: Long, + executorLogs: Map[String, String]) private[ui] class ExecutorsPage( - parent: ExecutorsTab, - threadDumpEnabled: Boolean) + parent: ExecutorsTab, + threadDumpEnabled: Boolean) extends WebUIPage("") { private val listener = parent.listener // When GCTimePercent is edited change ToolTips.TASK_TIME to match @@ -56,11 +56,9 @@ private[ui] class ExecutorsPage( def render(request: HttpServletRequest): Seq[Node] = { val content =
- { -
++ - ++ - - } + {
++ + ++ + }
; UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true) @@ -79,60 +77,59 @@ private[ui] class ExecutorsPage( } - {info.id} - {info.hostPort} + + {info.id} + + + {info.hostPort} + {executorStatus} - {info.rddBlocks} + + {info.rddBlocks} + - {Utils.bytesToString(memoryUsed)} / + {Utils.bytesToString(memoryUsed)} + / {Utils.bytesToString(maximumMemory)} {Utils.bytesToString(diskUsed)} - {info.totalCores} - {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, - info.totalTasks, info.totalDuration, info.totalGCTime)} - - {Utils.bytesToString(info.totalInputBytes)} - + + {info.totalCores} + {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, + info.totalTasks, info.totalDuration, info.totalGCTime)} + {Utils.bytesToString(info.totalInputBytes)} + {Utils.bytesToString(info.totalShuffleRead)} {Utils.bytesToString(info.totalShuffleWrite)} + {if (logsExist) { + + {info.executorLogs.map { case (logName, logUrl) => +
+ + {logName} + +
+ }} - { - if (logsExist) { - - { - info.executorLogs.map { case (logName, logUrl) => -
- - {logName} - -
- } - } - - } - } - { - if (threadDumpEnabled) { - if (info.isActive) { - val encodedId = URLEncoder.encode(info.id, "UTF-8") - - Thread Dump - - } else { - - } - } else { - Seq.empty - } + }}{if (threadDumpEnabled) { + if (info.isActive) { + val encodedId = URLEncoder.encode(info.id, "UTF-8") + + Thread Dump + + } else { + } + } else { + Seq.empty + }} } @@ -146,26 +143,35 @@ private[ui] class ExecutorsPage( val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum - {rowName}({execInfo.size}) - {execInfo.map(_.rddBlocks).sum} + + + {rowName} + ( + {execInfo.size} + ) + + + {execInfo.map(_.rddBlocks).sum} + - {Utils.bytesToString(memoryUsed)} / + {Utils.bytesToString(memoryUsed)} + / {Utils.bytesToString(maximumMemory)} {Utils.bytesToString(diskUsed)} - {totalCores} - {taskData(execInfo.map(_.maxTasks).sum, + + {totalCores} + {taskData(execInfo.map(_.maxTasks).sum, execInfo.map(_.activeTasks).sum, execInfo.map(_.failedTasks).sum, execInfo.map(_.completedTasks).sum, execInfo.map(_.totalTasks).sum, execInfo.map(_.totalDuration).sum, - execInfo.map(_.totalGCTime).sum)} - - {Utils.bytesToString(totalInputBytes)} - + execInfo.map(_.totalGCTime).sum)} + {Utils.bytesToString(totalInputBytes)} + {Utils.bytesToString(totalShuffleRead)} @@ -176,7 +182,7 @@ private[ui] class ExecutorsPage( } private def execSummary(activeExecInfo: Seq[ExecutorSummary], deadExecInfo: Seq[ExecutorSummary]): - Seq[Node] = { + Seq[Node] = { val totalExecInfo = activeExecInfo ++ deadExecInfo val activeRow = execSummaryRow(activeExecInfo, "Active"); val deadRow = execSummaryRow(deadExecInfo, "Dead"); @@ -186,16 +192,24 @@ private[ui] class ExecutorsPage( RDD Blocks - Storage Memory + + Storage Memory + Disk Used Cores Active Tasks Failed Tasks Complete Tasks Total Tasks - Task Time (GC Time) - Input - Shuffle Read + + Task Time (GC Time) + + + Input + + + Shuffle Read + Shuffle Write @@ -203,21 +217,19 @@ private[ui] class ExecutorsPage( - {activeRow} - {deadRow} - {totalRow} + {activeRow}{deadRow}{totalRow} } private def taskData( - maxTasks: Int, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalGCTime: Long): Seq[Node] = { + maxTasks: Int, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalGCTime: Long): Seq[Node] = { // Determine Color Opacity from 0.5-1 // activeTasks range from 0 to maxTasks val activeTasksAlpha = @@ -242,33 +254,37 @@ private[ui] class ExecutorsPage( } val tableData = - 0) { + 0) { "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" } else { "" - } - }>{activeTasks} - 0) { - "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" - } else { - "" - } - }>{failedTasks} - {completedTasks} - {totalTasks} - GCTimePercent * totalDuration) { - "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" - } else { - "" - } - }> - {Utils.msDurationToString(totalDuration)} - ({Utils.msDurationToString(totalGCTime)}) - ; + }}> + {activeTasks} + + 0) { + "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" + } else { + "" + }}> + {failedTasks} + + + {completedTasks} + + + {totalTasks} + + GCTimePercent * totalDuration) { + "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" + } else { + "" + }}> + {Utils.msDurationToString(totalDuration)} + ( + {Utils.msDurationToString(totalGCTime)} + ) + ; tableData } @@ -277,9 +293,9 @@ private[ui] class ExecutorsPage( private[spark] object ExecutorsPage { /** Represent an executor's info as a map given a storage status index */ def getExecInfo( - listener: ExecutorsListener, - statusId: Int, - isActive: Boolean): ExecutorSummary = { + listener: ExecutorsListener, + statusId: Int, + isActive: Boolean): ExecutorSummary = { val status = if (isActive) { listener.activeStorageStatusList(statusId) } else { From 43e4a87646cbdbcd043b7c3567e09082fffe2aad Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 13 Jul 2016 13:50:55 -0500 Subject: [PATCH 12/20] Remove unused method --- .../resources/org/apache/spark/ui/static/executorspage.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index ade0d71093c7..e6c58e3201a0 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -126,10 +126,6 @@ function failedTasksStyle(failedTasks, totalTasks) { ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; } -function failedTasksColor(failedTasks, totalTasks) { - return failedTasks > 0 ? "white" : "black"; -} - // totalDuration range from 0 to 50% GC time, alpha max = 1 function totalDurationAlpha(totalGCTime, totalDuration) { return totalDuration > 0 ? From fe12d599dc95dd7346f0f2f3ee04cdce039a05e7 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 13 Jul 2016 13:52:01 -0500 Subject: [PATCH 13/20] Remove unused variables --- .../main/resources/org/apache/spark/ui/static/executorspage.js | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index e6c58e3201a0..153a2b9a5fcb 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -155,7 +155,6 @@ $(document).ready(function () { }); executorsSummary = $("#active-executors"); - searchString = executorsSummary["context"]["location"]["search"]; var endPoint = createRESTEndPoint(); From 286d6d8b40e5e5840837e57c9e4a9aefaca19691 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 13 Jul 2016 13:55:10 -0500 Subject: [PATCH 14/20] Fix REST API documenation for all executors --- docs/monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/monitoring.md b/docs/monitoring.md index cae4a49c93c2..4aabc5d3e407 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -291,7 +291,7 @@ where `[base-app-id]` is the YARN application ID. A list of all active executors for the given application. - /applications/[app-id]/executors + /applications/[app-id]/allexecutors A list of all(active/dead) executors for the given application. From faf814fe5d710ecbc77d10cbf9028ca5f0aa5ebb Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 13 Jul 2016 14:41:41 -0500 Subject: [PATCH 15/20] Remove unwanted variables/methods from javascript --- .../apache/spark/ui/exec/ExecutorsPage.scala | 273 ++---------------- 1 file changed, 25 insertions(+), 248 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala index 80a3ab260e6b..287390b87bd7 100644 --- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala +++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala @@ -28,26 +28,26 @@ import org.apache.spark.util.Utils // This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive private[ui] case class ExecutorSummaryInfo( - id: String, - hostPort: String, - rddBlocks: Int, - memoryUsed: Long, - diskUsed: Long, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalInputBytes: Long, - totalShuffleRead: Long, - totalShuffleWrite: Long, - maxMemory: Long, - executorLogs: Map[String, String]) + id: String, + hostPort: String, + rddBlocks: Int, + memoryUsed: Long, + diskUsed: Long, + activeTasks: Int, + failedTasks: Int, + completedTasks: Int, + totalTasks: Int, + totalDuration: Long, + totalInputBytes: Long, + totalShuffleRead: Long, + totalShuffleWrite: Long, + maxMemory: Long, + executorLogs: Map[String, String]) private[ui] class ExecutorsPage( - parent: ExecutorsTab, - threadDumpEnabled: Boolean) + parent: ExecutorsTab, + threadDumpEnabled: Boolean) extends WebUIPage("") { private val listener = parent.listener // When GCTimePercent is edited change ToolTips.TASK_TIME to match @@ -56,246 +56,23 @@ private[ui] class ExecutorsPage( def render(request: HttpServletRequest): Seq[Node] = { val content =
- {
++ - ++ - } + { +
++ + ++ + + }
; UIUtils.headerSparkPage("Executors", content, parent, useDataTables = true) } - - /** Render an HTML row representing an executor */ - private def execRow(info: ExecutorSummary, logsExist: Boolean): Seq[Node] = { - val maximumMemory = info.maxMemory - val memoryUsed = info.memoryUsed - val diskUsed = info.diskUsed - val executorStatus = - if (info.isActive) { - "Active" - } else { - "Dead" - } - - - - {info.id} - - - {info.hostPort} - - - {executorStatus} - - - {info.rddBlocks} - - - {Utils.bytesToString(memoryUsed)} - / - {Utils.bytesToString(maximumMemory)} - - - {Utils.bytesToString(diskUsed)} - - - {info.totalCores} - {taskData(info.maxTasks, info.activeTasks, info.failedTasks, info.completedTasks, - info.totalTasks, info.totalDuration, info.totalGCTime)} - {Utils.bytesToString(info.totalInputBytes)} - - - {Utils.bytesToString(info.totalShuffleRead)} - - - {Utils.bytesToString(info.totalShuffleWrite)} - {if (logsExist) { - - {info.executorLogs.map { case (logName, logUrl) => - - }} - - }}{if (threadDumpEnabled) { - if (info.isActive) { - val encodedId = URLEncoder.encode(info.id, "UTF-8") - - Thread Dump - - } else { - - } - } else { - Seq.empty - }} - - } - - private def execSummaryRow(execInfo: Seq[ExecutorSummary], rowName: String): Seq[Node] = { - val maximumMemory = execInfo.map(_.maxMemory).sum - val memoryUsed = execInfo.map(_.memoryUsed).sum - val diskUsed = execInfo.map(_.diskUsed).sum - val totalCores = execInfo.map(_.totalCores).sum - val totalInputBytes = execInfo.map(_.totalInputBytes).sum - val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum - val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum - - - - - {rowName} - ( - {execInfo.size} - ) - - - {execInfo.map(_.rddBlocks).sum} - - - {Utils.bytesToString(memoryUsed)} - / - {Utils.bytesToString(maximumMemory)} - - - {Utils.bytesToString(diskUsed)} - - - {totalCores} - {taskData(execInfo.map(_.maxTasks).sum, - execInfo.map(_.activeTasks).sum, - execInfo.map(_.failedTasks).sum, - execInfo.map(_.completedTasks).sum, - execInfo.map(_.totalTasks).sum, - execInfo.map(_.totalDuration).sum, - execInfo.map(_.totalGCTime).sum)} - {Utils.bytesToString(totalInputBytes)} - - - {Utils.bytesToString(totalShuffleRead)} - - - {Utils.bytesToString(totalShuffleWrite)} - - - } - - private def execSummary(activeExecInfo: Seq[ExecutorSummary], deadExecInfo: Seq[ExecutorSummary]): - Seq[Node] = { - val totalExecInfo = activeExecInfo ++ deadExecInfo - val activeRow = execSummaryRow(activeExecInfo, "Active"); - val deadRow = execSummaryRow(deadExecInfo, "Dead"); - val totalRow = execSummaryRow(totalExecInfo, "Total"); - - - - - - - - - - - - - - - - - - - {activeRow}{deadRow}{totalRow} - -
RDD Blocks - Storage Memory - Disk UsedCoresActive TasksFailed TasksComplete TasksTotal Tasks - Task Time (GC Time) - - Input - - Shuffle Read - - - Shuffle Write - -
- } - - private def taskData( - maxTasks: Int, - activeTasks: Int, - failedTasks: Int, - completedTasks: Int, - totalTasks: Int, - totalDuration: Long, - totalGCTime: Long): Seq[Node] = { - // Determine Color Opacity from 0.5-1 - // activeTasks range from 0 to maxTasks - val activeTasksAlpha = - if (maxTasks > 0) { - (activeTasks.toDouble / maxTasks) * 0.5 + 0.5 - } else { - 1 - } - // failedTasks range max at 10% failure, alpha max = 1 - val failedTasksAlpha = - if (totalTasks > 0) { - math.min(10 * failedTasks.toDouble / totalTasks, 1) * 0.5 + 0.5 - } else { - 1 - } - // totalDuration range from 0 to 50% GC time, alpha max = 1 - val totalDurationAlpha = - if (totalDuration > 0) { - math.min(totalGCTime.toDouble / totalDuration + 0.5, 1) - } else { - 1 - } - - val tableData = - 0) { - "background:hsla(240, 100%, 50%, " + activeTasksAlpha + ");color:white" - } else { - "" - }}> - {activeTasks} - - 0) { - "background:hsla(0, 100%, 50%, " + failedTasksAlpha + ");color:white" - } else { - "" - }}> - {failedTasks} - - - {completedTasks} - - - {totalTasks} - - GCTimePercent * totalDuration) { - "background:hsla(0, 100%, 50%, " + totalDurationAlpha + ");color:white" - } else { - "" - }}> - {Utils.msDurationToString(totalDuration)} - ( - {Utils.msDurationToString(totalGCTime)} - ) - ; - - tableData - } } private[spark] object ExecutorsPage { /** Represent an executor's info as a map given a storage status index */ def getExecInfo( - listener: ExecutorsListener, - statusId: Int, - isActive: Boolean): ExecutorSummary = { + listener: ExecutorsListener, + statusId: Int, + isActive: Boolean): ExecutorSummary = { val status = if (isActive) { listener.activeStorageStatusList(statusId) } else { From abf68c78dbc51e28585066f6debe1bfc90a16c94 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Thu, 14 Jul 2016 14:48:39 -0500 Subject: [PATCH 16/20] Fix ExecutorsPage for standalone mode --- .../apache/spark/ui/static/executorspage.js | 627 +++++++++--------- 1 file changed, 330 insertions(+), 297 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 153a2b9a5fcb..f5abfb1fb1e9 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -55,31 +55,56 @@ $(document).ajaxStart(function () { $.blockUI({message: '

Loading Executors Page...

'}); }); -function createTemplateURI() { - var parser = document.createElement('a'); +function createTemplateURI(appId) { var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); if (ind > 0) { - var appId = words[ind + 1]; var baseURI = words.slice(0, ind + 1).join('/') + '/' + appId + '/static/executorspage-template.html'; return baseURI; - } else { - ind = words.indexOf("history"); + } + ind = words.indexOf("history"); + if(ind > 0) { var baseURI = words.slice(0, ind).join('/') + '/static/executorspage-template.html'; return baseURI; } + return location.origin + "/static/executorspage-template.html"; +} + +function getStandAloneppId(cb) { + var words = document.baseURI.split('/'); + var ind = words.indexOf("proxy"); + if (ind > 0) { + var appId = words[ind + 1]; + cb(appId); + return; + } + ind = words.indexOf("history"); + if (ind > 0) { + var appId = words[ind + 1]; + cb(appId); + return; + } + //Looks like Web UI is running in standalone mode + //Let's get application-id using REST End Point + $.getJSON(location.origin + "/api/v1/applications", function(response, status, jqXHR) { + if (response && response.length > 0) { + var appId = response[0].id + cb(appId); + return; + } + }); } -function createRESTEndPoint() { - var parser = document.createElement('a'); +function createRESTEndPoint(appId) { var words = document.baseURI.split('/'); var ind = words.indexOf("proxy"); if (ind > 0) { var appId = words[ind + 1]; var newBaseURI = words.slice(0, ind + 2).join('/'); return newBaseURI + "/api/v1/applications/" + appId + "/allexecutors" - } else { - ind = words.indexOf("history"); + } + ind = words.indexOf("history"); + if (ind > 0) { var appId = words[ind + 1]; var attemptId = words[ind + 2]; var newBaseURI = words.slice(0, ind).join('/'); @@ -89,6 +114,7 @@ function createRESTEndPoint() { return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/allexecutors"; } } + return location.origin + "/api/v1/applications/" + appId + "/allexecutors"; } function formatLogsCells(execLogs, type) { @@ -126,6 +152,10 @@ function failedTasksStyle(failedTasks, totalTasks) { ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; } +function failedTasksColor(failedTasks, totalTasks) { + return failedTasks > 0 ? "white" : "black"; +} + // totalDuration range from 0 to 50% GC time, alpha max = 1 function totalDurationAlpha(totalGCTime, totalDuration) { return totalDuration > 0 ? @@ -155,304 +185,307 @@ $(document).ready(function () { }); executorsSummary = $("#active-executors"); - - var endPoint = createRESTEndPoint(); - - $.getJSON(endPoint, function (response, status, jqXHR) { - var summary = []; - var allExecCnt = 0; - var allRDDBlocks = 0; - var allMemoryUsed = 0; - var allMaxMemory = 0; - var allDiskUsed = 0; - var allTotalCores = 0; - var allMaxTasks = 0; - var allActiveTasks = 0; - var allFailedTasks = 0; - var allCompletedTasks = 0; - var allTotalTasks = 0; - var allTotalDuration = 0; - var allTotalGCTime = 0; - var allTotalInputBytes = 0; - var allTotalShuffleRead = 0; - var allTotalShuffleWrite = 0; - - var activeExecCnt = 0; - var activeRDDBlocks = 0; - var activeMemoryUsed = 0; - var activeMaxMemory = 0; - var activeDiskUsed = 0; - var activeTotalCores = 0; - var activeMaxTasks = 0; - var activeActiveTasks = 0; - var activeFailedTasks = 0; - var activeCompletedTasks = 0; - var activeTotalTasks = 0; - var activeTotalDuration = 0; - var activeTotalGCTime = 0; - var activeTotalInputBytes = 0; - var activeTotalShuffleRead = 0; - var activeTotalShuffleWrite = 0; - - var deadExecCnt = 0; - var deadRDDBlocks = 0; - var deadMemoryUsed = 0; - var deadMaxMemory = 0; - var deadDiskUsed = 0; - var deadTotalCores = 0; - var deadMaxTasks = 0; - var deadActiveTasks = 0; - var deadFailedTasks = 0; - var deadCompletedTasks = 0; - var deadTotalTasks = 0; - var deadTotalDuration = 0; - var deadTotalGCTime = 0; - var deadTotalInputBytes = 0; - var deadTotalShuffleRead = 0; - var deadTotalShuffleWrite = 0; - - response.forEach(function (exec) { - allExecCnt += 1; - allRDDBlocks += exec.rddBlocks; - allMemoryUsed += exec.memoryUsed; - allMaxMemory += exec.maxMemory; - allDiskUsed += exec.diskUsed; - allTotalCores += exec.totalCores; - allMaxTasks += exec.maxTasks; - allActiveTasks += exec.activeTasks; - allFailedTasks += exec.failedTasks; - allCompletedTasks += exec.completedTasks; - allTotalTasks += exec.totalTasks; - allTotalDuration += exec.totalDuration; - allTotalGCTime += exec.totalGCTime; - allTotalInputBytes += exec.totalInputBytes; - allTotalShuffleRead += exec.totalShuffleRead; - allTotalShuffleWrite += exec.totalShuffleWrite; - if (exec.isActive) { - activeExecCnt += 1; - activeRDDBlocks += exec.rddBlocks; - activeMemoryUsed += exec.memoryUsed; - activeMaxMemory += exec.maxMemory; - activeDiskUsed += exec.diskUsed; - activeTotalCores += exec.totalCores; - activeMaxTasks += exec.maxTasks; - activeActiveTasks += exec.activeTasks; - activeFailedTasks += exec.failedTasks; - activeCompletedTasks += exec.completedTasks; - activeTotalTasks += exec.totalTasks; - activeTotalDuration += exec.totalDuration; - activeTotalGCTime += exec.totalGCTime; - activeTotalInputBytes += exec.totalInputBytes; - activeTotalShuffleRead += exec.totalShuffleRead; - activeTotalShuffleWrite += exec.totalShuffleWrite; - } else { - deadExecCnt += 1; - deadRDDBlocks += exec.rddBlocks; - deadMemoryUsed += exec.memoryUsed; - deadMaxMemory += exec.maxMemory; - deadDiskUsed += exec.diskUsed; - deadTotalCores += exec.totalCores; - deadMaxTasks += exec.maxTasks; - deadActiveTasks += exec.activeTasks; - deadFailedTasks += exec.failedTasks; - deadCompletedTasks += exec.completedTasks; - deadTotalTasks += exec.totalTasks; - deadTotalDuration += exec.totalDuration; - deadTotalGCTime += exec.totalGCTime; - deadTotalInputBytes += exec.totalInputBytes; - deadTotalShuffleRead += exec.totalShuffleRead; - deadTotalShuffleWrite += exec.totalShuffleWrite; - } - }); - - var totalSummary = { - "execCnt": ( "Total(" + allExecCnt + ")"), - "allRDDBlocks": allRDDBlocks, - "allMemoryUsed": allMemoryUsed, - "allMaxMemory": allMaxMemory, - "allDiskUsed": allDiskUsed, - "allTotalCores": allTotalCores, - "allMaxTasks": allMaxTasks, - "allActiveTasks": allActiveTasks, - "allFailedTasks": allFailedTasks, - "allCompletedTasks": allCompletedTasks, - "allTotalTasks": allTotalTasks, - "allTotalDuration": allTotalDuration, - "allTotalGCTime": allTotalGCTime, - "allTotalInputBytes": allTotalInputBytes, - "allTotalShuffleRead": allTotalShuffleRead, - "allTotalShuffleWrite": allTotalShuffleWrite - }; - var activeSummary = { - "execCnt": ( "Active(" + activeExecCnt + ")"), - "allRDDBlocks": activeRDDBlocks, - "allMemoryUsed": activeMemoryUsed, - "allMaxMemory": activeMaxMemory, - "allDiskUsed": activeDiskUsed, - "allTotalCores": activeTotalCores, - "allMaxTasks": activeMaxTasks, - "allActiveTasks": activeActiveTasks, - "allFailedTasks": activeFailedTasks, - "allCompletedTasks": activeCompletedTasks, - "allTotalTasks": activeTotalTasks, - "allTotalDuration": activeTotalDuration, - "allTotalGCTime": activeTotalGCTime, - "allTotalInputBytes": activeTotalInputBytes, - "allTotalShuffleRead": activeTotalShuffleRead, - "allTotalShuffleWrite": activeTotalShuffleWrite - }; - var deadSummary = { - "execCnt": ( "Dead(" + deadExecCnt + ")" ), - "allRDDBlocks": deadRDDBlocks, - "allMemoryUsed": deadMemoryUsed, - "allMaxMemory": deadMaxMemory, - "allDiskUsed": deadDiskUsed, - "allTotalCores": deadTotalCores, - "allMaxTasks": deadMaxTasks, - "allActiveTasks": deadActiveTasks, - "allFailedTasks": deadFailedTasks, - "allCompletedTasks": deadCompletedTasks, - "allTotalTasks": deadTotalTasks, - "allTotalDuration": deadTotalDuration, - "allTotalGCTime": deadTotalGCTime, - "allTotalInputBytes": deadTotalInputBytes, - "allTotalShuffleRead": deadTotalShuffleRead, - "allTotalShuffleWrite": deadTotalShuffleWrite - }; - - var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; - $.get(createTemplateURI(), function (template) { - - executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); - var selector = "#active-executors-table"; - var conf = { - "data": response, - "columns": [ - //{data: 'id'}, - { - data: function (row, type) { - return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; - } - }, - {data: 'hostPort'}, - {data: 'isActive', render: formatStatus}, - {data: 'rddBlocks'}, - { - data: function (row, type) { - return type === 'display' ? (formatBytes(row.memoryUsed, type) + ' / ' + formatBytes(row.maxMemory, type)) : row.memoryUsed; - } - }, - {data: 'diskUsed', render: formatBytes}, - {data: 'totalCores'}, - { - data: 'activeTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); + searchString = executorsSummary["context"]["location"]["search"]; + + getStandAloneppId(function (appId) { + + var endPoint = createRESTEndPoint(appId); + $.getJSON(endPoint, function (response, status, jqXHR) { + var summary = []; + var allExecCnt = 0; + var allRDDBlocks = 0; + var allMemoryUsed = 0; + var allMaxMemory = 0; + var allDiskUsed = 0; + var allTotalCores = 0; + var allMaxTasks = 0; + var allActiveTasks = 0; + var allFailedTasks = 0; + var allCompletedTasks = 0; + var allTotalTasks = 0; + var allTotalDuration = 0; + var allTotalGCTime = 0; + var allTotalInputBytes = 0; + var allTotalShuffleRead = 0; + var allTotalShuffleWrite = 0; + + var activeExecCnt = 0; + var activeRDDBlocks = 0; + var activeMemoryUsed = 0; + var activeMaxMemory = 0; + var activeDiskUsed = 0; + var activeTotalCores = 0; + var activeMaxTasks = 0; + var activeActiveTasks = 0; + var activeFailedTasks = 0; + var activeCompletedTasks = 0; + var activeTotalTasks = 0; + var activeTotalDuration = 0; + var activeTotalGCTime = 0; + var activeTotalInputBytes = 0; + var activeTotalShuffleRead = 0; + var activeTotalShuffleWrite = 0; + + var deadExecCnt = 0; + var deadRDDBlocks = 0; + var deadMemoryUsed = 0; + var deadMaxMemory = 0; + var deadDiskUsed = 0; + var deadTotalCores = 0; + var deadMaxTasks = 0; + var deadActiveTasks = 0; + var deadFailedTasks = 0; + var deadCompletedTasks = 0; + var deadTotalTasks = 0; + var deadTotalDuration = 0; + var deadTotalGCTime = 0; + var deadTotalInputBytes = 0; + var deadTotalShuffleRead = 0; + var deadTotalShuffleWrite = 0; + + response.forEach(function (exec) { + allExecCnt += 1; + allRDDBlocks += exec.rddBlocks; + allMemoryUsed += exec.memoryUsed; + allMaxMemory += exec.maxMemory; + allDiskUsed += exec.diskUsed; + allTotalCores += exec.totalCores; + allMaxTasks += exec.maxTasks; + allActiveTasks += exec.activeTasks; + allFailedTasks += exec.failedTasks; + allCompletedTasks += exec.completedTasks; + allTotalTasks += exec.totalTasks; + allTotalDuration += exec.totalDuration; + allTotalGCTime += exec.totalGCTime; + allTotalInputBytes += exec.totalInputBytes; + allTotalShuffleRead += exec.totalShuffleRead; + allTotalShuffleWrite += exec.totalShuffleWrite; + if (exec.isActive) { + activeExecCnt += 1; + activeRDDBlocks += exec.rddBlocks; + activeMemoryUsed += exec.memoryUsed; + activeMaxMemory += exec.maxMemory; + activeDiskUsed += exec.diskUsed; + activeTotalCores += exec.totalCores; + activeMaxTasks += exec.maxTasks; + activeActiveTasks += exec.activeTasks; + activeFailedTasks += exec.failedTasks; + activeCompletedTasks += exec.completedTasks; + activeTotalTasks += exec.totalTasks; + activeTotalDuration += exec.totalDuration; + activeTotalGCTime += exec.totalGCTime; + activeTotalInputBytes += exec.totalInputBytes; + activeTotalShuffleRead += exec.totalShuffleRead; + activeTotalShuffleWrite += exec.totalShuffleWrite; + } else { + deadExecCnt += 1; + deadRDDBlocks += exec.rddBlocks; + deadMemoryUsed += exec.memoryUsed; + deadMaxMemory += exec.maxMemory; + deadDiskUsed += exec.diskUsed; + deadTotalCores += exec.totalCores; + deadMaxTasks += exec.maxTasks; + deadActiveTasks += exec.activeTasks; + deadFailedTasks += exec.failedTasks; + deadCompletedTasks += exec.completedTasks; + deadTotalTasks += exec.totalTasks; + deadTotalDuration += exec.totalDuration; + deadTotalGCTime += exec.totalGCTime; + deadTotalInputBytes += exec.totalInputBytes; + deadTotalShuffleRead += exec.totalShuffleRead; + deadTotalShuffleWrite += exec.totalShuffleWrite; + } + }); + + var totalSummary = { + "execCnt": ( "Total(" + allExecCnt + ")"), + "allRDDBlocks": allRDDBlocks, + "allMemoryUsed": allMemoryUsed, + "allMaxMemory": allMaxMemory, + "allDiskUsed": allDiskUsed, + "allTotalCores": allTotalCores, + "allMaxTasks": allMaxTasks, + "allActiveTasks": allActiveTasks, + "allFailedTasks": allFailedTasks, + "allCompletedTasks": allCompletedTasks, + "allTotalTasks": allTotalTasks, + "allTotalDuration": allTotalDuration, + "allTotalGCTime": allTotalGCTime, + "allTotalInputBytes": allTotalInputBytes, + "allTotalShuffleRead": allTotalShuffleRead, + "allTotalShuffleWrite": allTotalShuffleWrite + }; + var activeSummary = { + "execCnt": ( "Active(" + activeExecCnt + ")"), + "allRDDBlocks": activeRDDBlocks, + "allMemoryUsed": activeMemoryUsed, + "allMaxMemory": activeMaxMemory, + "allDiskUsed": activeDiskUsed, + "allTotalCores": activeTotalCores, + "allMaxTasks": activeMaxTasks, + "allActiveTasks": activeActiveTasks, + "allFailedTasks": activeFailedTasks, + "allCompletedTasks": activeCompletedTasks, + "allTotalTasks": activeTotalTasks, + "allTotalDuration": activeTotalDuration, + "allTotalGCTime": activeTotalGCTime, + "allTotalInputBytes": activeTotalInputBytes, + "allTotalShuffleRead": activeTotalShuffleRead, + "allTotalShuffleWrite": activeTotalShuffleWrite + }; + var deadSummary = { + "execCnt": ( "Dead(" + deadExecCnt + ")" ), + "allRDDBlocks": deadRDDBlocks, + "allMemoryUsed": deadMemoryUsed, + "allMaxMemory": deadMaxMemory, + "allDiskUsed": deadDiskUsed, + "allTotalCores": deadTotalCores, + "allMaxTasks": deadMaxTasks, + "allActiveTasks": deadActiveTasks, + "allFailedTasks": deadFailedTasks, + "allCompletedTasks": deadCompletedTasks, + "allTotalTasks": deadTotalTasks, + "allTotalDuration": deadTotalDuration, + "allTotalGCTime": deadTotalGCTime, + "allTotalInputBytes": deadTotalInputBytes, + "allTotalShuffleRead": deadTotalShuffleRead, + "allTotalShuffleWrite": deadTotalShuffleWrite + }; + + var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]}; + $.get(createTemplateURI(appId), function (template) { + + executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data)); + var selector = "#active-executors-table"; + var conf = { + "data": response, + "columns": [ + //{data: 'id'}, + { + data: function (row, type) { + return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; } - } - }, - { - data: 'failedTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); + }, + {data: 'hostPort'}, + {data: 'isActive', render: formatStatus}, + {data: 'rddBlocks'}, + { + data: function (row, type) { + return type === 'display' ? (formatBytes(row.memoryUsed, type) + ' / ' + formatBytes(row.maxMemory, type)) : row.memoryUsed; } - } - }, - {data: 'completedTasks'}, - {data: 'totalTasks'}, - { - data: function (row, type) { - return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration }, - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (oData.totalDuration > 0) { - $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); - $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); + {data: 'diskUsed', render: formatBytes}, + {data: 'totalCores'}, + { + data: 'activeTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.activeTasks, oData.maxTasks)); + } } + }, + { + data: 'failedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.failedTasks, oData.totalTasks)); + } + } + }, + {data: 'completedTasks'}, + {data: 'totalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.totalDuration) + ' (' + formatDuration(row.totalGCTime) + ')') : row.totalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (oData.totalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.totalGCTime, oData.totalDuration)); + $(nTd).css('background', totalDurationStyle(oData.totalGCTime, oData.totalDuration)); + } + } + }, + {data: 'totalInputBytes', render: formatBytes}, + {data: 'totalShuffleRead', render: formatBytes}, + {data: 'totalShuffleWrite', render: formatBytes}, + {data: 'executorLogs', render: formatLogsCells}, + { + data: 'id', render: function (data, type) { + return type === 'display' ? ("Thread Dump" ) : data; } - }, - {data: 'totalInputBytes', render: formatBytes}, - {data: 'totalShuffleRead', render: formatBytes}, - {data: 'totalShuffleWrite', render: formatBytes}, - {data: 'executorLogs', render: formatLogsCells}, - { - data: 'id', render: function (data, type) { - return type === 'display' ? ("Thread Dump" ) : data; - } - } - ], - "order": [[0, "asc"]] - }; - - $(selector).DataTable(conf); - $('#active-executors [data-toggle="tooltip"]').tooltip(); - - var sumSelector = "#summary-execs-table"; - var sumConf = { - "data": [activeSummary, deadSummary, totalSummary], - "columns": [ - { - data: 'execCnt', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - $(nTd).css('font-weight', 'bold'); - } - }, - {data: 'allRDDBlocks'}, - { - data: function (row, type) { - return type === 'display' ? (formatBytes(row.allMemoryUsed, type) + ' / ' + formatBytes(row.allMaxMemory, type)) : row.allMemoryUsed; } - }, - {data: 'allDiskUsed', render: formatBytes}, - {data: 'allTotalCores'}, - { - data: 'allActiveTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); + ], + "order": [[0, "asc"]] + }; + + $(selector).DataTable(conf); + $('#active-executors [data-toggle="tooltip"]').tooltip(); + + var sumSelector = "#summary-execs-table"; + var sumConf = { + "data": [activeSummary, deadSummary, totalSummary], + "columns": [ + { + data: 'execCnt', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + $(nTd).css('font-weight', 'bold'); } - } - }, - { - data: 'allFailedTasks', - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (sData > 0) { - $(nTd).css('color', 'white'); - $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); + }, + {data: 'allRDDBlocks'}, + { + data: function (row, type) { + return type === 'display' ? (formatBytes(row.allMemoryUsed, type) + ' / ' + formatBytes(row.allMaxMemory, type)) : row.allMemoryUsed; } - } - }, - {data: 'allCompletedTasks'}, - {data: 'allTotalTasks'}, - { - data: function (row, type) { - return type === 'display' ? (formatDuration(row.allTotalDuration, type) + ' (' + formatDuration(row.allTotalGCTime, type) + ')') : row.allTotalDuration }, - "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { - if (oData.allTotalDuration > 0) { - $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); - $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); + {data: 'allDiskUsed', render: formatBytes}, + {data: 'allTotalCores'}, + { + data: 'allActiveTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', activeTasksStyle(oData.allActiveTasks, oData.allMaxTasks)); + } } - } - }, - {data: 'allTotalInputBytes', render: formatBytes}, - {data: 'allTotalShuffleRead', render: formatBytes}, - {data: 'allTotalShuffleWrite', render: formatBytes} - ], - "paging": false, - "searching": false, - "info": false - - }; - - $(sumSelector).DataTable(sumConf); - $('#execSummary [data-toggle="tooltip"]').tooltip(); - + }, + { + data: 'allFailedTasks', + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (sData > 0) { + $(nTd).css('color', 'white'); + $(nTd).css('background', failedTasksStyle(oData.allFailedTasks, oData.allTotalTasks)); + } + } + }, + {data: 'allCompletedTasks'}, + {data: 'allTotalTasks'}, + { + data: function (row, type) { + return type === 'display' ? (formatDuration(row.allTotalDuration, type) + ' (' + formatDuration(row.allTotalGCTime, type) + ')') : row.allTotalDuration + }, + "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) { + if (oData.allTotalDuration > 0) { + $(nTd).css('color', totalDurationColor(oData.allTotalGCTime, oData.allTotalDuration)); + $(nTd).css('background', totalDurationStyle(oData.allTotalGCTime, oData.allTotalDuration)); + } + } + }, + {data: 'allTotalInputBytes', render: formatBytes}, + {data: 'allTotalShuffleRead', render: formatBytes}, + {data: 'allTotalShuffleWrite', render: formatBytes} + ], + "paging": false, + "searching": false, + "info": false + + }; + + $(sumSelector).DataTable(sumConf); + $('#execSummary [data-toggle="tooltip"]').tooltip(); + + }); }); }); }); From bc132b1c23982062fbe3c411a746a9c0eb10b380 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Thu, 14 Jul 2016 14:50:27 -0500 Subject: [PATCH 17/20] Minor documentation fix --- docs/monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/monitoring.md b/docs/monitoring.md index 4aabc5d3e407..f9f83043dc9f 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -292,7 +292,7 @@ where `[base-app-id]` is the YARN application ID. /applications/[app-id]/allexecutors - A list of all(active/dead) executors for the given application. + A list of all(active and dead) executors for the given application. /applications/[app-id]/storage/rdd From 66df19fbf564f9541abebac4ecb0d00379bcb7ae Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Thu, 14 Jul 2016 15:35:58 -0500 Subject: [PATCH 18/20] Move formateBytes function to utils.js --- .../org/apache/spark/ui/static/executorspage.js | 11 ----------- .../resources/org/apache/spark/ui/static/utils.js | 10 ++++++++++ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index f5abfb1fb1e9..73e485e803de 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -24,17 +24,6 @@ function formatStatus(status, type) { } } -function formatBytes(bytes, type) { - if (type !== 'display') return bytes; - if (bytes == 0) return '0.0 B'; - var k = 1000; - var dm = 1; - var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; - var i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; -} - - jQuery.extend(jQuery.fn.dataTableExt.oSort, { "title-numeric-pre": function (a) { var x = a.match(/title="*(-?[0-9\.]+)/)[1]; diff --git a/core/src/main/resources/org/apache/spark/ui/static/utils.js b/core/src/main/resources/org/apache/spark/ui/static/utils.js index ae6c54ff9746..edc0ee2ce181 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/utils.js +++ b/core/src/main/resources/org/apache/spark/ui/static/utils.js @@ -36,3 +36,13 @@ function formatDuration(milliseconds) { var hours = minutes / 60; return hours.toFixed(1) + " h"; } + +function formatBytes(bytes, type) { + if (type !== 'display') return bytes; + if (bytes == 0) return '0.0 B'; + var k = 1000; + var dm = 1; + var sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + var i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; +} From e81d45bb7ba530ac37398e3e78cda8054123e10f Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Mon, 18 Jul 2016 10:14:03 -0500 Subject: [PATCH 19/20] Remove commented unused js lines --- .../main/resources/org/apache/spark/ui/static/executorspage.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index 73e485e803de..f15c02f6c650 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -174,7 +174,6 @@ $(document).ready(function () { }); executorsSummary = $("#active-executors"); - searchString = executorsSummary["context"]["location"]["search"]; getStandAloneppId(function (appId) { @@ -349,7 +348,6 @@ $(document).ready(function () { var conf = { "data": response, "columns": [ - //{data: 'id'}, { data: function (row, type) { return type !== 'display' ? (isNaN(row.id) ? 0 : row.id ) : row.id; From 87301c1e0de1ade000fb7d05685b61ca87fc1491 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 20 Jul 2016 14:47:25 +0000 Subject: [PATCH 20/20] Remove unused javascript functions --- .../resources/org/apache/spark/ui/static/executorspage.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js index f15c02f6c650..b2b2363d3ac6 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js @@ -126,10 +126,6 @@ function activeTasksStyle(activeTasks, maxTasks) { return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; } -function activeTasksColor(activeTasks, maxTasks) { - return activeTasks > 0 ? ("hsla(240, 100%, 50%, " + activeTasksAlpha(activeTasks, maxTasks) + ")") : ""; -} - // failedTasks range max at 10% failure, alpha max = 1 function failedTasksAlpha(failedTasks, totalTasks) { return totalTasks > 0 ? @@ -141,10 +137,6 @@ function failedTasksStyle(failedTasks, totalTasks) { ("hsla(0, 100%, 50%, " + failedTasksAlpha(failedTasks, totalTasks) + ")") : ""; } -function failedTasksColor(failedTasks, totalTasks) { - return failedTasks > 0 ? "white" : "black"; -} - // totalDuration range from 0 to 50% GC time, alpha max = 1 function totalDurationAlpha(totalGCTime, totalDuration) { return totalDuration > 0 ?