Skip to content

Commit 098a93d

Browse files
author
pgandhi
committed
[SPARK-21809] : Change Stage Page to use datatables to support sorting columns and searching
[SPARK-21809] : Fixed a couple of ui issues; the changes are visible both in webui and SHS when running on yarn. Fixed unit tests. Removed two obsolete unit tests.
1 parent dd12be7 commit 098a93d

14 files changed

+1098
-286
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ <h4 id="tasksTitle" class="title-table"></h4>
9898
<th>GC Time</span></th>
9999
<th>Scheduler Delay</span></th>
100100
<th>Task Deserialization Time</th>
101+
<th>Shuffle Read Blocked Time</th>
102+
<th>Shuffle Remote Reads</th>
101103
<th>Result Serialization Time</th>
102104
<th>Getting Result Time</th>
103105
<th>Peak Execution Memory</th>

‎core/src/main/resources/org/apache/spark/ui/static/taskspages.js‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,12 @@ function StageEndPoint(appId) {
6666
ind = words.indexOf("history");
6767
if (ind > 0) {
6868
var appId = words[ind + 1];
69-
var attemptId = words[ind + 4].split("&attempt=").pop();
70-
var stageIdLen = words[ind + 4].indexOf('&');
71-
var stageId = words[ind + 4].substr(4, stageIdLen - 4);
69+
var attemptId = words[ind + 5].split("&attempt=").pop();
70+
var stageIdLen = words[ind + 5].indexOf('&');
71+
var stageId = words[ind + 5].substr(4, stageIdLen - 4);
7272
var newBaseURI = words.slice(0, ind).join('/');
7373
if (isNaN(attemptId) || attemptId == "0") {
74-
return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
74+
return newBaseURI + "/api/v1/applications/" + appId + "/" + "1" + "/stages/" + stageId;
7575
} else {
7676
return newBaseURI + "/api/v1/applications/" + appId + "/" + attemptId + "/stages/" + stageId;
7777
}
@@ -110,12 +110,14 @@ $(document).ready(function () {
110110
" Show Additional Metrics" +
111111
"</a></div>" +
112112
"<div class='container-fluid' id='toggle-metrics' hidden>" +
113-
"<div><input type='checkbox' class='toggle-vis' data-column='0'> Select All</div>" +
114-
"<div><input type='checkbox' class='toggle-vis' data-column='10'> Scheduler Delay</div>" +
115-
"<div><input type='checkbox' class='toggle-vis' data-column='11'> Task Deserialization Time</div>" +
116-
"<div><input type='checkbox' class='toggle-vis' data-column='12'> Result Serialization Time</div>" +
117-
"<div><input type='checkbox' class='toggle-vis' data-column='13'> Getting Result Time</div>" +
118-
"<div><input type='checkbox' class='toggle-vis' data-column='14'> Peak Execution Memory</div>" +
113+
"<div><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
114+
"<div><input type='checkbox' class='toggle-vis' id='box-10' data-column='10'> Scheduler Delay</div>" +
115+
"<div><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Task Deserialization Time</div>" +
116+
"<div><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Shuffle Read Blocked Time</div>" +
117+
"<div><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Remote Reads</div>" +
118+
"<div><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Result Serialization Time</div>" +
119+
"<div><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Getting Result Time</div>" +
120+
"<div><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Peak Execution Memory</div>" +
119121
"</div>");
120122

121123
tasksSummary = $("#active-tasks");
@@ -372,6 +374,16 @@ $(document).ready(function () {
372374
return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
373375
}
374376
},
377+
{
378+
data : function (row, type) {
379+
return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
380+
}
381+
},
382+
{
383+
data : function (row, type) {
384+
return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
385+
}
386+
},
375387
{
376388
data : function (row, type) {
377389
return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
@@ -384,7 +396,7 @@ $(document).ready(function () {
384396
},
385397
{
386398
data : function (row, type) {
387-
return type === 'display' ? formatDuration(row.taskMetrics.peakExecutionMemory) : row.taskMetrics.peakExecutionMemory;
399+
return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
388400
}
389401
},
390402
{
@@ -406,18 +418,33 @@ $(document).ready(function () {
406418
{ "visible": false, "targets": 11 },
407419
{ "visible": false, "targets": 12 },
408420
{ "visible": false, "targets": 13 },
409-
{ "visible": false, "targets": 14 }
421+
{ "visible": false, "targets": 14 },
422+
{ "visible": false, "targets": 15 },
423+
{ "visible": false, "targets": 16 }
410424
],
411425
"order": [[0, "asc"]]
412426
};
413427
var taskTableSelector = $(taskTable).DataTable(task_conf);
414428

429+
var optionalColumns = [10, 11, 12, 13, 14, 15, 16];
430+
var allChecked = true;
431+
for(k = 0; k < optionalColumns.length; k++) {
432+
if (taskTableSelector.column(optionalColumns[k]).visible()) {
433+
document.getElementById("box-"+optionalColumns[k]).checked = true;
434+
} else {
435+
allChecked = false;
436+
}
437+
}
438+
if (allChecked) {
439+
document.getElementById("box-0").checked = true;
440+
}
441+
415442
// hide or show columns dynamically event
416443
$('input.toggle-vis').on('click', function(e){
417444
// Get the column
418445
var para = $(this).attr('data-column');
419446
if(para == "0"){
420-
var column = taskTableSelector.column([10, 11, 12, 13, 14]);
447+
var column = taskTableSelector.column([10, 11, 12, 13, 14, 15, 16]);
421448
if($(this).is(":checked")){
422449
$(".toggle-vis").prop('checked', true);
423450
column.visible(true);

‎core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json‎

Lines changed: 57 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,21 @@
3131
"attempt" : 0,
3232
"launchTime" : "2015-02-03T16:43:05.829GMT",
3333
"duration" : 435,
34+
"gettingResultTime" : 0.0,
35+
"schedulerDelay" : 16.0,
3436
"executorId" : "<driver>",
3537
"host" : "localhost",
3638
"status" : "SUCCESS",
3739
"taskLocality" : "PROCESS_LOCAL",
40+
"taskState" : "SUCCESS",
3841
"speculative" : false,
3942
"accumulatorUpdates" : [ ],
4043
"taskMetrics" : {
4144
"executorDeserializeTime" : 1,
4245
"executorDeserializeCpuTime" : 0,
4346
"executorRunTime" : 435,
4447
"executorCpuTime" : 0,
48+
"peakExecutionMemory" : 0,
4549
"resultSize" : 1902,
4650
"jvmGcTime" : 19,
4751
"resultSerializationTime" : 2,
@@ -69,25 +73,30 @@
6973
"writeTime" : 94000,
7074
"recordsWritten" : 0
7175
}
72-
}
76+
},
77+
"executorLogs" : { }
7378
},
7479
"9" : {
7580
"taskId" : 9,
7681
"index" : 1,
7782
"attempt" : 0,
7883
"launchTime" : "2015-02-03T16:43:05.830GMT",
7984
"duration" : 436,
85+
"gettingResultTime" : 0.0,
86+
"schedulerDelay" : 17.0,
8087
"executorId" : "<driver>",
8188
"host" : "localhost",
8289
"status" : "SUCCESS",
8390
"taskLocality" : "PROCESS_LOCAL",
91+
"taskState" : "SUCCESS",
8492
"speculative" : false,
8593
"accumulatorUpdates" : [ ],
8694
"taskMetrics" : {
8795
"executorDeserializeTime" : 1,
8896
"executorDeserializeCpuTime" : 0,
8997
"executorRunTime" : 436,
9098
"executorCpuTime" : 0,
99+
"peakExecutionMemory" : 0,
91100
"resultSize" : 1902,
92101
"jvmGcTime" : 19,
93102
"resultSerializationTime" : 0,
@@ -115,25 +124,30 @@
115124
"writeTime" : 98000,
116125
"recordsWritten" : 0
117126
}
118-
}
127+
},
128+
"executorLogs" : { }
119129
},
120130
"10" : {
121131
"taskId" : 10,
122132
"index" : 2,
123133
"attempt" : 0,
124134
"launchTime" : "2015-02-03T16:43:05.830GMT",
125135
"duration" : 434,
136+
"gettingResultTime" : 0.0,
137+
"schedulerDelay" : 19.0,
126138
"executorId" : "<driver>",
127139
"host" : "localhost",
128140
"status" : "SUCCESS",
129141
"taskLocality" : "PROCESS_LOCAL",
142+
"taskState" : "SUCCESS",
130143
"speculative" : false,
131144
"accumulatorUpdates" : [ ],
132145
"taskMetrics" : {
133146
"executorDeserializeTime" : 2,
134147
"executorDeserializeCpuTime" : 0,
135148
"executorRunTime" : 434,
136149
"executorCpuTime" : 0,
150+
"peakExecutionMemory" : 0,
137151
"resultSize" : 1902,
138152
"jvmGcTime" : 19,
139153
"resultSerializationTime" : 1,
@@ -161,25 +175,30 @@
161175
"writeTime" : 76000,
162176
"recordsWritten" : 0
163177
}
164-
}
178+
},
179+
"executorLogs" : { }
165180
},
166181
"11" : {
167182
"taskId" : 11,
168183
"index" : 3,
169184
"attempt" : 0,
170185
"launchTime" : "2015-02-03T16:43:05.830GMT",
171186
"duration" : 434,
187+
"gettingResultTime" : 0.0,
188+
"schedulerDelay" : 17.0,
172189
"executorId" : "<driver>",
173190
"host" : "localhost",
174191
"status" : "SUCCESS",
175192
"taskLocality" : "PROCESS_LOCAL",
193+
"taskState" : "SUCCESS",
176194
"speculative" : false,
177195
"accumulatorUpdates" : [ ],
178196
"taskMetrics" : {
179197
"executorDeserializeTime" : 2,
180198
"executorDeserializeCpuTime" : 0,
181199
"executorRunTime" : 434,
182200
"executorCpuTime" : 0,
201+
"peakExecutionMemory" : 0,
183202
"resultSize" : 1902,
184203
"jvmGcTime" : 19,
185204
"resultSerializationTime" : 1,
@@ -207,25 +226,30 @@
207226
"writeTime" : 83000,
208227
"recordsWritten" : 0
209228
}
210-
}
229+
},
230+
"executorLogs" : { }
211231
},
212232
"12" : {
213233
"taskId" : 12,
214234
"index" : 4,
215235
"attempt" : 0,
216236
"launchTime" : "2015-02-03T16:43:05.831GMT",
217237
"duration" : 434,
238+
"gettingResultTime" : 0.0,
239+
"schedulerDelay" : 17.0,
218240
"executorId" : "<driver>",
219241
"host" : "localhost",
220242
"status" : "SUCCESS",
221243
"taskLocality" : "PROCESS_LOCAL",
244+
"taskState" : "SUCCESS",
222245
"speculative" : false,
223246
"accumulatorUpdates" : [ ],
224247
"taskMetrics" : {
225248
"executorDeserializeTime" : 2,
226249
"executorDeserializeCpuTime" : 0,
227250
"executorRunTime" : 434,
228251
"executorCpuTime" : 0,
252+
"peakExecutionMemory" : 0,
229253
"resultSize" : 1902,
230254
"jvmGcTime" : 19,
231255
"resultSerializationTime" : 1,
@@ -253,25 +277,30 @@
253277
"writeTime" : 101000,
254278
"recordsWritten" : 0
255279
}
256-
}
280+
},
281+
"executorLogs" : { }
257282
},
258283
"13" : {
259284
"taskId" : 13,
260285
"index" : 5,
261286
"attempt" : 0,
262287
"launchTime" : "2015-02-03T16:43:05.831GMT",
263288
"duration" : 434,
289+
"gettingResultTime" : 0.0,
290+
"schedulerDelay" : 14.0,
264291
"executorId" : "<driver>",
265292
"host" : "localhost",
266293
"status" : "SUCCESS",
267294
"taskLocality" : "PROCESS_LOCAL",
295+
"taskState" : "SUCCESS",
268296
"speculative" : false,
269297
"accumulatorUpdates" : [ ],
270298
"taskMetrics" : {
271299
"executorDeserializeTime" : 2,
272300
"executorDeserializeCpuTime" : 0,
273301
"executorRunTime" : 434,
274302
"executorCpuTime" : 0,
303+
"peakExecutionMemory" : 0,
275304
"resultSize" : 1902,
276305
"jvmGcTime" : 19,
277306
"resultSerializationTime" : 2,
@@ -299,25 +328,30 @@
299328
"writeTime" : 73000,
300329
"recordsWritten" : 0
301330
}
302-
}
331+
},
332+
"executorLogs" : { }
303333
},
304334
"14" : {
305335
"taskId" : 14,
306336
"index" : 6,
307337
"attempt" : 0,
308338
"launchTime" : "2015-02-03T16:43:05.832GMT",
309339
"duration" : 434,
340+
"gettingResultTime" : 0.0,
341+
"schedulerDelay" : 13.0,
310342
"executorId" : "<driver>",
311343
"host" : "localhost",
312344
"status" : "SUCCESS",
313345
"taskLocality" : "PROCESS_LOCAL",
346+
"taskState" : "SUCCESS",
314347
"speculative" : false,
315348
"accumulatorUpdates" : [ ],
316349
"taskMetrics" : {
317350
"executorDeserializeTime" : 2,
318351
"executorDeserializeCpuTime" : 0,
319352
"executorRunTime" : 434,
320353
"executorCpuTime" : 0,
354+
"peakExecutionMemory" : 0,
321355
"resultSize" : 1902,
322356
"jvmGcTime" : 19,
323357
"resultSerializationTime" : 1,
@@ -345,25 +379,30 @@
345379
"writeTime" : 88000,
346380
"recordsWritten" : 0
347381
}
348-
}
382+
},
383+
"executorLogs" : { }
349384
},
350385
"15" : {
351386
"taskId" : 15,
352387
"index" : 7,
353388
"attempt" : 0,
354389
"launchTime" : "2015-02-03T16:43:05.833GMT",
355390
"duration" : 435,
391+
"gettingResultTime" : 0.0,
392+
"schedulerDelay" : 13.0,
356393
"executorId" : "<driver>",
357394
"host" : "localhost",
358395
"status" : "SUCCESS",
359396
"taskLocality" : "PROCESS_LOCAL",
397+
"taskState" : "SUCCESS",
360398
"speculative" : false,
361399
"accumulatorUpdates" : [ ],
362400
"taskMetrics" : {
363401
"executorDeserializeTime" : 1,
364402
"executorDeserializeCpuTime" : 0,
365403
"executorRunTime" : 435,
366404
"executorCpuTime" : 0,
405+
"peakExecutionMemory" : 0,
367406
"resultSize" : 1902,
368407
"jvmGcTime" : 19,
369408
"resultSerializationTime" : 1,
@@ -391,7 +430,8 @@
391430
"writeTime" : 79000,
392431
"recordsWritten" : 0
393432
}
394-
}
433+
},
434+
"executorLogs" : { }
395435
}
396436
},
397437
"executorSummary" : {
@@ -403,8 +443,16 @@
403443
"outputBytes" : 0,
404444
"shuffleRead" : 0,
405445
"shuffleWrite" : 13180,
446+
"shuffleWriteRecords" : 0,
406447
"memoryBytesSpilled" : 0,
407-
"diskBytesSpilled" : 0
448+
"diskBytesSpilled" : 0,
449+
"killedTasks" : 0,
450+
"blacklisted" : 0,
451+
"inputRecords" : 0,
452+
"outputRecords" : 0,
453+
"shuffleReadRecords" : 0,
454+
"host" : "localhost:57971",
455+
"executorLogs" : { }
408456
}
409457
}
410458
}

0 commit comments

Comments
 (0)