From 908bafd26fe15e20667b8575242ec8ea4c8fdec0 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 13 Dec 2018 21:21:25 +0800 Subject: [PATCH 1/4] remove executorLogs in TaskData --- .../org/apache/spark/ui/static/stagepage.js | 14 ++++- .../apache/spark/status/AppStatusStore.scala | 7 --- .../org/apache/spark/status/api/v1/api.scala | 1 - .../org/apache/spark/status/storeTypes.scala | 1 - .../blacklisting_for_stage_expectation.json | 48 ---------------- ...acklisting_node_for_stage_expectation.json | 56 ------------------- .../one_stage_attempt_json_expectation.json | 8 --- .../one_stage_json_expectation.json | 8 --- .../stage_task_list_expectation.json | 20 ------- ...multi_attempt_app_json_1__expectation.json | 8 --- ...multi_attempt_app_json_2__expectation.json | 8 --- ...k_list_w__offset___length_expectation.json | 50 ----------------- ...stage_task_list_w__sortBy_expectation.json | 20 ------- ...tBy_short_names___runtime_expectation.json | 20 ------- ...rtBy_short_names__runtime_expectation.json | 20 ------- ...age_with_accumulable_json_expectation.json | 8 --- .../spark/status/AppStatusUtilsSuite.scala | 2 - 17 files changed, 12 insertions(+), 287 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js index 08de2b0fee03..71ff5934a474 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js @@ -348,9 +348,9 @@ $(document).ready(function () { // prepare data for executor summary table stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary); + var executorDetailsMap = {}; $.getJSON(createRESTEndPointForExecutorsPage(appId), function(executorSummaryResponse, status, jqXHR) { - var executorDetailsMap = {}; executorSummaryResponse.forEach(function (executorDetail) { executorDetailsMap[executorDetail.id] = executorDetail; }); @@ -664,7 +664,17 @@ $(document).ready(function () { {data : "taskLocality", name: "Locality Level"}, {data : "executorId", name: "Executor ID"}, {data : "host", name: "Host"}, - {data : "executorLogs", name: "Logs", render: formatLogsCells}, + { + data : function (row, type) { + if(executorDetailsMap[row.executorId] && executorDetailsMap[row.executorId]["executorLogs"]) { + return executorDetailsMap[row.executorId]["executorLogs"]; + } else { + return ""; + } + }, + name: "Logs", + render: formatLogsCells + }, {data : "launchTime", name: "Launch Time", render: formatDate}, { data : function (row, type) { diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index 312bcccb1cca..abddc73c006e 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -538,18 +538,11 @@ private[spark] class AppStatusStore( def constructTaskData(taskDataWrapper: TaskDataWrapper) : v1.TaskData = { val taskDataOld: v1.TaskData = taskDataWrapper.toApi - val executorLogs: Option[Map[String, String]] = try { - Some(executorSummary(taskDataOld.executorId).executorLogs) - } catch { - case e: NoSuchElementException => e.getMessage - None - } new v1.TaskData(taskDataOld.taskId, taskDataOld.index, taskDataOld.attempt, taskDataOld.launchTime, taskDataOld.resultFetchStart, taskDataOld.duration, taskDataOld.executorId, taskDataOld.host, taskDataOld.status, taskDataOld.taskLocality, taskDataOld.speculative, taskDataOld.accumulatorUpdates, taskDataOld.errorMessage, taskDataOld.taskMetrics, - executorLogs.getOrElse(Map[String, String]()), AppStatusUtils.schedulerDelay(taskDataOld), AppStatusUtils.gettingResultTime(taskDataOld)) } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index c7d3cd37db6f..443159e0e80e 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -254,7 +254,6 @@ class TaskData private[spark]( val accumulatorUpdates: Seq[AccumulableInfo], val errorMessage: Option[String] = None, val taskMetrics: Option[TaskMetrics] = None, - val executorLogs: Map[String, String], val schedulerDelay: Long, val gettingResultTime: Long) diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala index eea47b3b1709..7f8739d1a5c6 100644 --- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala +++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala @@ -285,7 +285,6 @@ private[spark] class TaskDataWrapper( accumulatorUpdates, errorMessage, metrics, - executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) } diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json index 62e5c123fd3d..b4f6984cf2a8 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json @@ -76,10 +76,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout", - "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr" - }, "schedulerDelay" : 102, "gettingResultTime" : 0 }, @@ -129,10 +125,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, @@ -182,10 +174,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, @@ -235,10 +223,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 51, "gettingResultTime" : 0 }, @@ -288,10 +272,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -341,10 +321,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -395,10 +371,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout", - "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr" - }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, @@ -448,10 +420,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, @@ -501,10 +469,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, @@ -554,10 +518,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, @@ -607,10 +567,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -660,10 +616,6 @@ "recordsWritten" : 1 } }, - "executorLogs" : { - "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", - "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" - }, "schedulerDelay" : 5, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json index 6e46c881b2a2..3d1707f6fc50 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json @@ -75,10 +75,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 68, "gettingResultTime" : 0 }, @@ -129,10 +125,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 46, "gettingResultTime" : 0 }, @@ -182,10 +174,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 50, "gettingResultTime" : 0 }, @@ -236,10 +224,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 174, "gettingResultTime" : 0 }, @@ -290,10 +274,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -343,10 +323,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 42, "gettingResultTime" : 0 }, @@ -396,10 +372,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 26, "gettingResultTime" : 0 }, @@ -449,10 +421,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 74, "gettingResultTime" : 0 }, @@ -502,10 +470,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -555,10 +519,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, @@ -608,10 +568,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 63, "gettingResultTime" : 0 }, @@ -661,10 +617,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 12, "gettingResultTime" : 0 }, @@ -714,10 +666,6 @@ "recordsWritten" : 3 } }, - "executorLogs" : { - "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 84, "gettingResultTime" : 0 }, @@ -768,10 +716,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { - "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096", - "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096" - }, "schedulerDelay" : 338, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index aa9471301fe3..7f4cadf1e09e 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -75,7 +75,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 19, "gettingResultTime" : 0 }, @@ -125,7 +124,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, @@ -175,7 +173,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -225,7 +222,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, @@ -275,7 +271,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -325,7 +320,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -375,7 +369,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -425,7 +418,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 584803b5e863..2da031376add 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -75,7 +75,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 19, "gettingResultTime" : 0 }, @@ -125,7 +124,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, @@ -175,7 +173,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -225,7 +222,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, @@ -275,7 +271,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -325,7 +320,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -375,7 +369,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -425,7 +418,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json index f859ab6fff24..10c223e6a9fb 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json @@ -44,7 +44,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -93,7 +92,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -142,7 +140,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -191,7 +188,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -240,7 +236,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -289,7 +284,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -338,7 +332,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -387,7 +380,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -436,7 +428,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -485,7 +476,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -534,7 +524,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -583,7 +572,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -632,7 +620,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -681,7 +668,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -730,7 +716,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -779,7 +764,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -828,7 +812,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -877,7 +860,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -926,7 +908,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -975,7 +956,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json index ea88ca116707..373321d7675c 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json @@ -49,7 +49,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, { @@ -103,7 +102,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -157,7 +155,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -211,7 +208,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -265,7 +261,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -319,7 +314,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, { @@ -373,7 +367,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -427,7 +420,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json index efd0a45bf01d..0a3e346d64da 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json @@ -49,7 +49,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, { @@ -103,7 +102,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -157,7 +155,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -211,7 +208,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -265,7 +261,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -319,7 +314,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, { @@ -373,7 +367,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -427,7 +420,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json index d83528d84972..a887ad70be78 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json @@ -44,7 +44,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -93,7 +92,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -142,7 +140,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -191,7 +188,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -240,7 +236,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -289,7 +284,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -338,7 +332,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -387,7 +380,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -436,7 +428,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -485,7 +476,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -534,7 +524,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -583,7 +572,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -632,7 +620,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -681,7 +668,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -730,7 +716,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -779,7 +764,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 9, "gettingResultTime" : 0 }, { @@ -828,7 +812,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -877,7 +860,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -926,7 +908,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, { @@ -975,7 +956,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, { @@ -1024,7 +1004,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -1073,7 +1052,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, { @@ -1122,7 +1100,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1171,7 +1148,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, { @@ -1220,7 +1196,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 9, "gettingResultTime" : 0 }, { @@ -1269,7 +1244,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, { @@ -1318,7 +1292,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1367,7 +1340,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1416,7 +1388,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1465,7 +1436,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1514,7 +1484,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1563,7 +1532,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1612,7 +1580,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 15, "gettingResultTime" : 0 }, { @@ -1661,7 +1628,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 21, "gettingResultTime" : 0 }, { @@ -1710,7 +1676,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -1759,7 +1724,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -1808,7 +1772,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, { @@ -1857,7 +1820,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1906,7 +1868,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -1955,7 +1916,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -2004,7 +1964,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -2053,7 +2012,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -2102,7 +2060,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2151,7 +2108,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2200,7 +2156,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -2249,7 +2204,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2298,7 +2252,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -2347,7 +2300,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -2396,7 +2348,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -2445,7 +2396,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json index 82e339c8f56d..695e385dae86 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json @@ -44,7 +44,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -93,7 +92,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -142,7 +140,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -191,7 +188,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -240,7 +236,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -289,7 +284,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -338,7 +332,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -387,7 +380,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -436,7 +428,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -485,7 +476,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -534,7 +524,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -583,7 +572,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -632,7 +620,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -681,7 +668,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -730,7 +716,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -779,7 +764,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -828,7 +812,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -877,7 +860,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -926,7 +908,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -975,7 +956,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json index 82e339c8f56d..695e385dae86 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json @@ -44,7 +44,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -93,7 +92,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -142,7 +140,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -191,7 +188,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -240,7 +236,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -289,7 +284,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -338,7 +332,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -387,7 +380,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -436,7 +428,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -485,7 +476,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -534,7 +524,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -583,7 +572,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -632,7 +620,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -681,7 +668,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -730,7 +716,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -779,7 +764,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -828,7 +812,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -877,7 +860,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -926,7 +908,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -975,7 +956,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json index 01eef1b565bf..68debb02b583 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json @@ -44,7 +44,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -93,7 +92,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -142,7 +140,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 21, "gettingResultTime" : 0 }, { @@ -191,7 +188,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -240,7 +236,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -289,7 +284,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -338,7 +332,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -387,7 +380,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -436,7 +428,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -485,7 +476,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 15, "gettingResultTime" : 0 }, { @@ -534,7 +524,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -583,7 +572,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -632,7 +620,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -681,7 +668,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -730,7 +716,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -779,7 +764,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -828,7 +812,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -877,7 +860,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -926,7 +908,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -975,7 +956,6 @@ "recordsWritten" : 10 } }, - "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index a8e1fd303a42..98812baaf998 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -84,7 +84,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, @@ -139,7 +138,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, @@ -194,7 +192,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -249,7 +246,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -304,7 +300,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, @@ -359,7 +354,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -414,7 +408,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, @@ -469,7 +462,6 @@ "recordsWritten" : 0 } }, - "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 } diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala index a01b24d323d2..b23c246d7af0 100644 --- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala @@ -53,7 +53,6 @@ class AppStatusUtilsSuite extends SparkFunSuite { outputMetrics = null, shuffleReadMetrics = null, shuffleWriteMetrics = null)), - executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) assert(AppStatusUtils.schedulerDelay(runningTask) === 0L) @@ -87,7 +86,6 @@ class AppStatusUtilsSuite extends SparkFunSuite { outputMetrics = null, shuffleReadMetrics = null, shuffleWriteMetrics = null)), - executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) assert(AppStatusUtils.schedulerDelay(finishedTask) === 3L) From e89040f6fc7c9ea62728b3d2cba7c54c71dcc0c8 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 21 Dec 2018 00:00:08 +0800 Subject: [PATCH 2/4] Revert "remove executorLogs in TaskData" This reverts commit 1cfd872c9f64b214ef4a3a17b507033bf296d60b. --- .../org/apache/spark/ui/static/stagepage.js | 14 +---- .../apache/spark/status/AppStatusStore.scala | 7 +++ .../org/apache/spark/status/api/v1/api.scala | 1 + .../org/apache/spark/status/storeTypes.scala | 1 + .../blacklisting_for_stage_expectation.json | 48 ++++++++++++++++ ...acklisting_node_for_stage_expectation.json | 56 +++++++++++++++++++ .../one_stage_attempt_json_expectation.json | 8 +++ .../one_stage_json_expectation.json | 8 +++ .../stage_task_list_expectation.json | 20 +++++++ ...multi_attempt_app_json_1__expectation.json | 8 +++ ...multi_attempt_app_json_2__expectation.json | 8 +++ ...k_list_w__offset___length_expectation.json | 50 +++++++++++++++++ ...stage_task_list_w__sortBy_expectation.json | 20 +++++++ ...tBy_short_names___runtime_expectation.json | 20 +++++++ ...rtBy_short_names__runtime_expectation.json | 20 +++++++ ...age_with_accumulable_json_expectation.json | 8 +++ .../spark/status/AppStatusUtilsSuite.scala | 2 + 17 files changed, 287 insertions(+), 12 deletions(-) diff --git a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js index 71ff5934a474..08de2b0fee03 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/stagepage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/stagepage.js @@ -348,9 +348,9 @@ $(document).ready(function () { // prepare data for executor summary table stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary); - var executorDetailsMap = {}; $.getJSON(createRESTEndPointForExecutorsPage(appId), function(executorSummaryResponse, status, jqXHR) { + var executorDetailsMap = {}; executorSummaryResponse.forEach(function (executorDetail) { executorDetailsMap[executorDetail.id] = executorDetail; }); @@ -664,17 +664,7 @@ $(document).ready(function () { {data : "taskLocality", name: "Locality Level"}, {data : "executorId", name: "Executor ID"}, {data : "host", name: "Host"}, - { - data : function (row, type) { - if(executorDetailsMap[row.executorId] && executorDetailsMap[row.executorId]["executorLogs"]) { - return executorDetailsMap[row.executorId]["executorLogs"]; - } else { - return ""; - } - }, - name: "Logs", - render: formatLogsCells - }, + {data : "executorLogs", name: "Logs", render: formatLogsCells}, {data : "launchTime", name: "Launch Time", render: formatDate}, { data : function (row, type) { diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index abddc73c006e..312bcccb1cca 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -538,11 +538,18 @@ private[spark] class AppStatusStore( def constructTaskData(taskDataWrapper: TaskDataWrapper) : v1.TaskData = { val taskDataOld: v1.TaskData = taskDataWrapper.toApi + val executorLogs: Option[Map[String, String]] = try { + Some(executorSummary(taskDataOld.executorId).executorLogs) + } catch { + case e: NoSuchElementException => e.getMessage + None + } new v1.TaskData(taskDataOld.taskId, taskDataOld.index, taskDataOld.attempt, taskDataOld.launchTime, taskDataOld.resultFetchStart, taskDataOld.duration, taskDataOld.executorId, taskDataOld.host, taskDataOld.status, taskDataOld.taskLocality, taskDataOld.speculative, taskDataOld.accumulatorUpdates, taskDataOld.errorMessage, taskDataOld.taskMetrics, + executorLogs.getOrElse(Map[String, String]()), AppStatusUtils.schedulerDelay(taskDataOld), AppStatusUtils.gettingResultTime(taskDataOld)) } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 443159e0e80e..c7d3cd37db6f 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -254,6 +254,7 @@ class TaskData private[spark]( val accumulatorUpdates: Seq[AccumulableInfo], val errorMessage: Option[String] = None, val taskMetrics: Option[TaskMetrics] = None, + val executorLogs: Map[String, String], val schedulerDelay: Long, val gettingResultTime: Long) diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala index 7f8739d1a5c6..eea47b3b1709 100644 --- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala +++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala @@ -285,6 +285,7 @@ private[spark] class TaskDataWrapper( accumulatorUpdates, errorMessage, metrics, + executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) } diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json index b4f6984cf2a8..62e5c123fd3d 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_for_stage_expectation.json @@ -76,6 +76,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout", + "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr" + }, "schedulerDelay" : 102, "gettingResultTime" : 0 }, @@ -125,6 +129,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, @@ -174,6 +182,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, @@ -223,6 +235,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 51, "gettingResultTime" : 0 }, @@ -272,6 +288,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -321,6 +341,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -371,6 +395,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stdout", + "stderr" : "http://172.30.65.138:64279/logPage/?appId=app-20180109111548-0000&executorId=0&logType=stderr" + }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, @@ -420,6 +448,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, @@ -469,6 +501,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, @@ -518,6 +554,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, @@ -567,6 +607,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, @@ -616,6 +660,10 @@ "recordsWritten" : 1 } }, + "executorLogs" : { + "stdout" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stdout", + "stderr" : "http://172.30.65.138:64278/logPage/?appId=app-20180109111548-0000&executorId=1&logType=stderr" + }, "schedulerDelay" : 5, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json index 3d1707f6fc50..6e46c881b2a2 100644 --- a/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/blacklisting_node_for_stage_expectation.json @@ -75,6 +75,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 68, "gettingResultTime" : 0 }, @@ -125,6 +129,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 46, "gettingResultTime" : 0 }, @@ -174,6 +182,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 50, "gettingResultTime" : 0 }, @@ -224,6 +236,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000007/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 174, "gettingResultTime" : 0 }, @@ -274,6 +290,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -323,6 +343,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 42, "gettingResultTime" : 0 }, @@ -372,6 +396,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 26, "gettingResultTime" : 0 }, @@ -421,6 +449,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000004/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 74, "gettingResultTime" : 0 }, @@ -470,6 +502,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -519,6 +555,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, @@ -568,6 +608,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000003/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 63, "gettingResultTime" : 0 }, @@ -617,6 +661,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 12, "gettingResultTime" : 0 }, @@ -666,6 +714,10 @@ "recordsWritten" : 3 } }, + "executorLogs" : { + "stdout" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-3.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000002/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 84, "gettingResultTime" : 0 }, @@ -716,6 +768,10 @@ "recordsWritten" : 0 } }, + "executorLogs" : { + "stdout" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stdout?start=-4096", + "stderr" : "http://apiros-2.gce.test.com:8042/node/containerlogs/container_1516285256255_0012_01_000005/attilapiros/stderr?start=-4096" + }, "schedulerDelay" : 338, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 7f4cadf1e09e..aa9471301fe3 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -75,6 +75,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 19, "gettingResultTime" : 0 }, @@ -124,6 +125,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, @@ -173,6 +175,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -222,6 +225,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, @@ -271,6 +275,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -320,6 +325,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -369,6 +375,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -418,6 +425,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 2da031376add..584803b5e863 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -75,6 +75,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 19, "gettingResultTime" : 0 }, @@ -124,6 +125,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, @@ -173,6 +175,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -222,6 +225,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, @@ -271,6 +275,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -320,6 +325,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, @@ -369,6 +375,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, @@ -418,6 +425,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json index 10c223e6a9fb..f859ab6fff24 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json @@ -44,6 +44,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -92,6 +93,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -140,6 +142,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -188,6 +191,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -236,6 +240,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -284,6 +289,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -332,6 +338,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -380,6 +387,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -428,6 +436,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -476,6 +485,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -524,6 +534,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -572,6 +583,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -620,6 +632,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -668,6 +681,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -716,6 +730,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -764,6 +779,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -812,6 +828,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -860,6 +877,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -908,6 +926,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -956,6 +975,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json index 373321d7675c..ea88ca116707 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json @@ -49,6 +49,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, { @@ -102,6 +103,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -155,6 +157,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -208,6 +211,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -261,6 +265,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -314,6 +319,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, { @@ -367,6 +373,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -420,6 +427,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json index 0a3e346d64da..efd0a45bf01d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json @@ -49,6 +49,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, { @@ -102,6 +103,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -155,6 +157,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -208,6 +211,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -261,6 +265,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -314,6 +319,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, { @@ -367,6 +373,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, { @@ -420,6 +427,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json index a887ad70be78..d83528d84972 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json @@ -44,6 +44,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, { @@ -92,6 +93,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -140,6 +142,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -188,6 +191,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -236,6 +240,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -284,6 +289,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -332,6 +338,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -380,6 +387,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -428,6 +436,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -476,6 +485,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -524,6 +534,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -572,6 +583,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -620,6 +632,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -668,6 +681,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -716,6 +730,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -764,6 +779,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 9, "gettingResultTime" : 0 }, { @@ -812,6 +828,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -860,6 +877,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -908,6 +926,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 17, "gettingResultTime" : 0 }, { @@ -956,6 +975,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, { @@ -1004,6 +1024,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 }, { @@ -1052,6 +1073,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 14, "gettingResultTime" : 0 }, { @@ -1100,6 +1122,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1148,6 +1171,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, { @@ -1196,6 +1220,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 9, "gettingResultTime" : 0 }, { @@ -1244,6 +1269,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 10, "gettingResultTime" : 0 }, { @@ -1292,6 +1318,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1340,6 +1367,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1388,6 +1416,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1436,6 +1465,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1484,6 +1514,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1532,6 +1563,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -1580,6 +1612,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 15, "gettingResultTime" : 0 }, { @@ -1628,6 +1661,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 21, "gettingResultTime" : 0 }, { @@ -1676,6 +1710,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -1724,6 +1759,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -1772,6 +1808,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 11, "gettingResultTime" : 0 }, { @@ -1820,6 +1857,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -1868,6 +1906,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -1916,6 +1955,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -1964,6 +2004,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -2012,6 +2053,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -2060,6 +2102,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2108,6 +2151,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2156,6 +2200,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -2204,6 +2249,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -2252,6 +2298,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -2300,6 +2347,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -2348,6 +2396,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -2396,6 +2445,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json index 695e385dae86..82e339c8f56d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json @@ -44,6 +44,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -92,6 +93,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -140,6 +142,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -188,6 +191,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -236,6 +240,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -284,6 +289,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -332,6 +338,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -380,6 +387,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -428,6 +436,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -476,6 +485,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -524,6 +534,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -572,6 +583,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -620,6 +632,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -668,6 +681,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -716,6 +730,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -764,6 +779,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -812,6 +828,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -860,6 +877,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -908,6 +926,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -956,6 +975,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json index 695e385dae86..82e339c8f56d 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json @@ -44,6 +44,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -92,6 +93,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 33, "gettingResultTime" : 0 }, { @@ -140,6 +142,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 40, "gettingResultTime" : 0 }, { @@ -188,6 +191,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 43, "gettingResultTime" : 0 }, { @@ -236,6 +240,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 38, "gettingResultTime" : 0 }, { @@ -284,6 +289,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 41, "gettingResultTime" : 0 }, { @@ -332,6 +338,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 53, "gettingResultTime" : 0 }, { @@ -380,6 +387,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 37, "gettingResultTime" : 0 }, { @@ -428,6 +436,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -476,6 +485,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -524,6 +534,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, { @@ -572,6 +583,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -620,6 +632,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -668,6 +681,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -716,6 +730,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -764,6 +779,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -812,6 +828,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -860,6 +877,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 }, { @@ -908,6 +926,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -956,6 +975,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json index 68debb02b583..01eef1b565bf 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json @@ -44,6 +44,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -92,6 +93,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 6, "gettingResultTime" : 0 }, { @@ -140,6 +142,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 21, "gettingResultTime" : 0 }, { @@ -188,6 +191,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -236,6 +240,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -284,6 +289,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -332,6 +338,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 8, "gettingResultTime" : 0 }, { @@ -380,6 +387,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -428,6 +436,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -476,6 +485,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 15, "gettingResultTime" : 0 }, { @@ -524,6 +534,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 2, "gettingResultTime" : 0 }, { @@ -572,6 +583,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -620,6 +632,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -668,6 +681,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 7, "gettingResultTime" : 0 }, { @@ -716,6 +730,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -764,6 +779,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 3, "gettingResultTime" : 0 }, { @@ -812,6 +828,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 16, "gettingResultTime" : 0 }, { @@ -860,6 +877,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 13, "gettingResultTime" : 0 }, { @@ -908,6 +926,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 4, "gettingResultTime" : 0 }, { @@ -956,6 +975,7 @@ "recordsWritten" : 10 } }, + "executorLogs" : { }, "schedulerDelay" : 5, "gettingResultTime" : 0 } ] diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index 98812baaf998..a8e1fd303a42 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -84,6 +84,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 30, "gettingResultTime" : 0 }, @@ -138,6 +139,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 23, "gettingResultTime" : 0 }, @@ -192,6 +194,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -246,6 +249,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -300,6 +304,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 18, "gettingResultTime" : 0 }, @@ -354,6 +359,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 22, "gettingResultTime" : 0 }, @@ -408,6 +414,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 20, "gettingResultTime" : 0 }, @@ -462,6 +469,7 @@ "recordsWritten" : 0 } }, + "executorLogs" : { }, "schedulerDelay" : 24, "gettingResultTime" : 0 } diff --git a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala index b23c246d7af0..a01b24d323d2 100644 --- a/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/status/AppStatusUtilsSuite.scala @@ -53,6 +53,7 @@ class AppStatusUtilsSuite extends SparkFunSuite { outputMetrics = null, shuffleReadMetrics = null, shuffleWriteMetrics = null)), + executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) assert(AppStatusUtils.schedulerDelay(runningTask) === 0L) @@ -86,6 +87,7 @@ class AppStatusUtilsSuite extends SparkFunSuite { outputMetrics = null, shuffleReadMetrics = null, shuffleWriteMetrics = null)), + executorLogs = null, schedulerDelay = 0L, gettingResultTime = 0L) assert(AppStatusUtils.schedulerDelay(finishedTask) === 3L) From 67992a5e7402e8a0c269e323ea3923292d27ebc4 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 21 Dec 2018 01:10:38 +0800 Subject: [PATCH 3/4] use hash map to reduce kv store lookup --- .../apache/spark/status/AppStatusStore.scala | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index 312bcccb1cca..9fd55c6480f0 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -20,6 +20,7 @@ package org.apache.spark.status import java.util.{List => JList} import scala.collection.JavaConverters._ +import scala.collection.mutable.HashMap import org.apache.spark.{JobExecutionStatus, SparkConf} import org.apache.spark.status.api.v1 @@ -386,10 +387,9 @@ private[spark] class AppStatusStore( def taskList(stageId: Int, stageAttemptId: Int, maxTasks: Int): Seq[v1.TaskData] = { val stageKey = Array(stageId, stageAttemptId) - store.view(classOf[TaskDataWrapper]).index("stage").first(stageKey).last(stageKey).reverse() - .max(maxTasks).asScala.map { taskDataWrapper => - constructTaskData(taskDataWrapper) - }.toSeq.reverse + val taskDataWrapperIter = store.view(classOf[TaskDataWrapper]).index("stage") + .first(stageKey).last(stageKey).reverse().max(maxTasks).asScala + constructTaskDataList(taskDataWrapperIter).reverse } def taskList( @@ -428,9 +428,8 @@ private[spark] class AppStatusStore( } val ordered = if (ascending) indexed else indexed.reverse() - ordered.skip(offset).max(length).asScala.map { taskDataWrapper => - constructTaskData(taskDataWrapper) - }.toSeq + val taskDataWrapperIter = ordered.skip(offset).max(length).asScala + constructTaskDataList(taskDataWrapperIter) } def executorSummary(stageId: Int, attemptId: Int): Map[String, v1.ExecutorStageSummary] = { @@ -554,6 +553,29 @@ private[spark] class AppStatusStore( AppStatusUtils.gettingResultTime(taskDataOld)) } + def constructTaskDataList(taskDataWrapperIter: Iterable[TaskDataWrapper]): Seq[v1.TaskData] = { + val executorIdToLogs = new HashMap[String, Map[String, String]]() + taskDataWrapperIter.map { taskDataWrapper => + val taskDataOld: v1.TaskData = taskDataWrapper.toApi + val executorLogs = executorIdToLogs.getOrElseUpdate(taskDataOld.executorId, { + try { + executorSummary(taskDataOld.executorId).executorLogs + } catch { + case e: NoSuchElementException => + Map.empty + } + }) + + new v1.TaskData(taskDataOld.taskId, taskDataOld.index, + taskDataOld.attempt, taskDataOld.launchTime, taskDataOld.resultFetchStart, + taskDataOld.duration, taskDataOld.executorId, taskDataOld.host, taskDataOld.status, + taskDataOld.taskLocality, taskDataOld.speculative, taskDataOld.accumulatorUpdates, + taskDataOld.errorMessage, taskDataOld.taskMetrics, + executorLogs, + AppStatusUtils.schedulerDelay(taskDataOld), + AppStatusUtils.gettingResultTime(taskDataOld)) + }.toSeq + } } private[spark] object AppStatusStore { From 40582c13df9512718a026ceaf80484f5dc759e74 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Fri, 21 Dec 2018 01:17:06 +0800 Subject: [PATCH 4/4] remove unused code --- .../apache/spark/status/AppStatusStore.scala | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala index 9fd55c6480f0..0487f2f07c09 100644 --- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala +++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala @@ -535,24 +535,6 @@ private[spark] class AppStatusStore( store.close() } - def constructTaskData(taskDataWrapper: TaskDataWrapper) : v1.TaskData = { - val taskDataOld: v1.TaskData = taskDataWrapper.toApi - val executorLogs: Option[Map[String, String]] = try { - Some(executorSummary(taskDataOld.executorId).executorLogs) - } catch { - case e: NoSuchElementException => e.getMessage - None - } - new v1.TaskData(taskDataOld.taskId, taskDataOld.index, - taskDataOld.attempt, taskDataOld.launchTime, taskDataOld.resultFetchStart, - taskDataOld.duration, taskDataOld.executorId, taskDataOld.host, taskDataOld.status, - taskDataOld.taskLocality, taskDataOld.speculative, taskDataOld.accumulatorUpdates, - taskDataOld.errorMessage, taskDataOld.taskMetrics, - executorLogs.getOrElse(Map[String, String]()), - AppStatusUtils.schedulerDelay(taskDataOld), - AppStatusUtils.gettingResultTime(taskDataOld)) - } - def constructTaskDataList(taskDataWrapperIter: Iterable[TaskDataWrapper]): Seq[v1.TaskData] = { val executorIdToLogs = new HashMap[String, Map[String, String]]() taskDataWrapperIter.map { taskDataWrapper =>