Skip to content

Commit d79eb80

Browse files
author
pgandhi
committed
[SPARK-21809] : Refactoring Code and adding shufflereadmetrics data
1 parent 590e93f commit d79eb80

File tree

4 files changed

+127
-56
lines changed

4 files changed

+127
-56
lines changed

‎core/src/main/resources/org/apache/spark/ui/static/stagepage.js‎

Lines changed: 108 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ function getColumnNameForTaskMetricSummary(columnKey) {
9898
else if(columnKey == "memoryBytesSpilled") {
9999
return "Shuffle spill (memory)";
100100
}
101+
else if(columnKey == "shuffleReadMetrics") {
102+
return "Shuffle Read Size / Records";
103+
}
101104
else if(columnKey == "shuffleWriteMetrics") {
102105
return "Shuffle Write Size / Records";
103106
}
@@ -176,7 +179,24 @@ $(document).ready(function () {
176179
var taskMetricIndices = Object.keys(response1);
177180
taskMetricIndices.forEach(function (ix) {
178181
var columnName = getColumnNameForTaskMetricSummary(ix);
179-
if (columnName != "NA") {
182+
if (columnName == "Shuffle Read Size / Records") {
183+
var row1 = {
184+
"metric": columnName,
185+
"data": response1[ix]
186+
};
187+
var row2 = {
188+
"metric": "Shuffle Read Blocked Time",
189+
"data": response1[ix]
190+
};
191+
var row3 = {
192+
"metric": "Shuffle Remote Reads",
193+
"data": response1[ix]
194+
};
195+
task_metrics_table.push(row1);
196+
task_metrics_table.push(row2);
197+
task_metrics_table.push(row3);
198+
}
199+
else if (columnName != "NA") {
180200
var row = {
181201
"metric": columnName,
182202
"data": response1[ix]
@@ -202,6 +222,19 @@ $(document).ready(function () {
202222
var str2arr = JSON.stringify(row.data.recordsWritten).split("[")[1].split("]")[0].split(",");
203223
var str = formatBytes(str1arr[0], type) + " / " + str2arr[0];
204224
return str;
225+
} else if (row.metric == 'Shuffle Read Size / Records') {
226+
var str1arr = JSON.stringify(row.data.readBytes).split("[")[1].split("]")[0].split(",");
227+
var str2arr = JSON.stringify(row.data.readRecords).split("[")[1].split("]")[0].split(",");
228+
var str = formatBytes(str1arr[0], type) + " / " + str2arr[0];
229+
return str;
230+
} else if (row.metric == 'Shuffle Read Blocked Time') {
231+
var str1arr = JSON.stringify(row.data.fetchWaitTime).split("[")[1].split("]")[0].split(",");
232+
var str = formatDuration(str1arr[0]);
233+
return str;
234+
} else if (row.metric == 'Shuffle Remote Reads') {
235+
var str1arr = JSON.stringify(row.data.remoteBytesRead).split("[")[1].split("]")[0].split(",");
236+
var str = formatBytes(str1arr[0], type);
237+
return str;
205238
} else if (row.metric == 'Shuffle Write Size / Records') {
206239
var str1arr = JSON.stringify(row.data.writeBytes).split("[")[1].split("]")[0].split(",");
207240
var str2arr = JSON.stringify(row.data.writeRecords).split("[")[1].split("]")[0].split(",");
@@ -225,6 +258,19 @@ $(document).ready(function () {
225258
var str2arr = JSON.stringify(row.data.recordsWritten).split("[")[1].split("]")[0].split(",");
226259
var str = formatBytes(str1arr[1], type) + " / " + str2arr[1];
227260
return str;
261+
} else if (row.metric == 'Shuffle Read Size / Records') {
262+
var str1arr = JSON.stringify(row.data.readBytes).split("[")[1].split("]")[0].split(",");
263+
var str2arr = JSON.stringify(row.data.readRecords).split("[")[1].split("]")[0].split(",");
264+
var str = formatBytes(str1arr[1], type) + " / " + str2arr[1];
265+
return str;
266+
} else if (row.metric == 'Shuffle Read Blocked Time') {
267+
var str1arr = JSON.stringify(row.data.fetchWaitTime).split("[")[1].split("]")[0].split(",");
268+
var str = formatDuration(str1arr[1]);
269+
return str;
270+
} else if (row.metric == 'Shuffle Remote Reads') {
271+
var str1arr = JSON.stringify(row.data.remoteBytesRead).split("[")[1].split("]")[0].split(",");
272+
var str = formatBytes(str1arr[1], type);
273+
return str;
228274
} else if (row.metric == 'Shuffle Write Size / Records') {
229275
var str1arr = JSON.stringify(row.data.writeBytes).split("[")[1].split("]")[0].split(",");
230276
var str2arr = JSON.stringify(row.data.writeRecords).split("[")[1].split("]")[0].split(",");
@@ -248,6 +294,19 @@ $(document).ready(function () {
248294
var str2arr = JSON.stringify(row.data.recordsWritten).split("[")[1].split("]")[0].split(",");
249295
var str = formatBytes(str1arr[2], type) + " / " + str2arr[2];
250296
return str;
297+
} else if (row.metric == 'Shuffle Read Size / Records') {
298+
var str1arr = JSON.stringify(row.data.readBytes).split("[")[1].split("]")[0].split(",");
299+
var str2arr = JSON.stringify(row.data.readRecords).split("[")[1].split("]")[0].split(",");
300+
var str = formatBytes(str1arr[2], type) + " / " + str2arr[2];
301+
return str;
302+
} else if (row.metric == 'Shuffle Read Blocked Time') {
303+
var str1arr = JSON.stringify(row.data.fetchWaitTime).split("[")[1].split("]")[0].split(",");
304+
var str = formatDuration(str1arr[2]);
305+
return str;
306+
} else if (row.metric == 'Shuffle Remote Reads') {
307+
var str1arr = JSON.stringify(row.data.remoteBytesRead).split("[")[1].split("]")[0].split(",");
308+
var str = formatBytes(str1arr[2], type);
309+
return str;
251310
} else if (row.metric == 'Shuffle Write Size / Records') {
252311
var str1arr = JSON.stringify(row.data.writeBytes).split("[")[1].split("]")[0].split(",");
253312
var str2arr = JSON.stringify(row.data.writeRecords).split("[")[1].split("]")[0].split(",");
@@ -271,6 +330,19 @@ $(document).ready(function () {
271330
var str2arr = JSON.stringify(row.data.recordsWritten).split("[")[1].split("]")[0].split(",");
272331
var str = formatBytes(str1arr[3], type) + " / " + str2arr[3];
273332
return str;
333+
} else if (row.metric == 'Shuffle Read Size / Records') {
334+
var str1arr = JSON.stringify(row.data.readBytes).split("[")[1].split("]")[0].split(",");
335+
var str2arr = JSON.stringify(row.data.readRecords).split("[")[1].split("]")[0].split(",");
336+
var str = formatBytes(str1arr[3], type) + " / " + str2arr[3];
337+
return str;
338+
} else if (row.metric == 'Shuffle Read Blocked Time') {
339+
var str1arr = JSON.stringify(row.data.fetchWaitTime).split("[")[1].split("]")[0].split(",");
340+
var str = formatDuration(str1arr[3]);
341+
return str;
342+
} else if (row.metric == 'Shuffle Remote Reads') {
343+
var str1arr = JSON.stringify(row.data.remoteBytesRead).split("[")[1].split("]")[0].split(",");
344+
var str = formatBytes(str1arr[3], type);
345+
return str;
274346
} else if (row.metric == 'Shuffle Write Size / Records') {
275347
var str1arr = JSON.stringify(row.data.writeBytes).split("[")[1].split("]")[0].split(",");
276348
var str2arr = JSON.stringify(row.data.writeRecords).split("[")[1].split("]")[0].split(",");
@@ -294,7 +366,20 @@ $(document).ready(function () {
294366
var str2arr = JSON.stringify(row.data.recordsWritten).split("[")[1].split("]")[0].split(",");
295367
var str = formatBytes(str1arr[4], type) + " / " + str2arr[4];
296368
return str;
297-
} else if (row.metric == 'Shuffle Write Size / Records') {
369+
} else if (row.metric == 'Shuffle Read Size / Records') {
370+
var str1arr = JSON.stringify(row.data.readBytes).split("[")[1].split("]")[0].split(",");
371+
var str2arr = JSON.stringify(row.data.readRecords).split("[")[1].split("]")[0].split(",");
372+
var str = formatBytes(str1arr[4], type) + " / " + str2arr[4];
373+
return str;
374+
} else if (row.metric == 'Shuffle Read Blocked Time') {
375+
var str1arr = JSON.stringify(row.data.fetchWaitTime).split("[")[1].split("]")[0].split(",");
376+
var str = formatDuration(str1arr[4]);
377+
return str;
378+
} else if (row.metric == 'Shuffle Remote Reads') {
379+
var str1arr = JSON.stringify(row.data.remoteBytesRead).split("[")[1].split("]")[0].split(",");
380+
var str = formatBytes(str1arr[4], type);
381+
return str;
382+
} else if (row.metric == 'Shuffle Write Size / Records') {
298383
var str1arr = JSON.stringify(row.data.writeBytes).split("[")[1].split("]")[0].split(",");
299384
var str2arr = JSON.stringify(row.data.writeRecords).split("[")[1].split("]")[0].split(",");
300385
var str = formatBytes(str1arr[4], type) + " / " + str2arr[4];
@@ -394,14 +479,7 @@ $(document).ready(function () {
394479
}
395480
$(accumulatorTable).DataTable(accumulator_conf);
396481

397-
var showInputMetrics = false;
398-
var showOutputMetrics = false;
399-
var showShuffleWriteTimeMetrics = false;
400-
var showShuffleWriteBytesMetrics = false;
401-
var showDiskSpill = false;
402-
var showMemSpill = false;
403-
404-
// building tasks table
482+
// building tasks table that uses server side functionality
405483
var taskTable = "#active-tasks-table";
406484
var task_conf = {
407485
"serverSide": true,
@@ -534,7 +612,6 @@ $(document).ready(function () {
534612
data : function (row, type) {
535613
if ("taskMetrics" in row) {
536614
if (row.taskMetrics.inputMetrics.bytesRead > 0) {
537-
showInputMetrics = true;
538615
if (type === 'display') {
539616
return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
540617
} else {
@@ -553,7 +630,6 @@ $(document).ready(function () {
553630
data : function (row, type) {
554631
if ("taskMetrics" in row) {
555632
if (row.taskMetrics.outputMetrics.bytesWritten > 0) {
556-
showOutputMetrics = true;
557633
if (type === 'display') {
558634
return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
559635
} else {
@@ -572,7 +648,6 @@ $(document).ready(function () {
572648
data : function (row, type) {
573649
if ("taskMetrics" in row) {
574650
if (row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
575-
showShuffleWriteTimeMetrics = true;
576651
return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000) : row.taskMetrics.shuffleWriteMetrics.writeTime;
577652
} else {
578653
return "";
@@ -587,7 +662,6 @@ $(document).ready(function () {
587662
data : function (row, type) {
588663
if ("taskMetrics" in row) {
589664
if (row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
590-
showShuffleWriteBytesMetrics = true;
591665
if (type === 'display') {
592666
return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
593667
} else {
@@ -602,11 +676,29 @@ $(document).ready(function () {
602676
},
603677
name: "Shuffle Write Size / Records"
604678
},
679+
{
680+
data : function (row, type) {
681+
if ("taskMetrics" in row) {
682+
if (row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
683+
var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
684+
if (type === 'display') {
685+
return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
686+
} else {
687+
return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
688+
}
689+
} else {
690+
return "";
691+
}
692+
} else {
693+
return "";
694+
}
695+
},
696+
name: "Shuffle Read Size / Records"
697+
},
605698
{
606699
data : function (row, type) {
607700
if ("taskMetrics" in row) {
608701
if (row.taskMetrics.memoryBytesSpilled > 0) {
609-
showMemSpill = true;
610702
return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
611703
} else {
612704
return "";
@@ -621,7 +713,6 @@ $(document).ready(function () {
621713
data : function (row, type) {
622714
if ("taskMetrics" in row) {
623715
if (row.taskMetrics.diskBytesSpilled > 0) {
624-
showDiskSpill = true;
625716
return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
626717
} else {
627718
return "";
@@ -655,13 +746,7 @@ $(document).ready(function () {
655746
{ "visible": false, "targets": 15 },
656747
{ "visible": false, "targets": 16 },
657748
{ "visible": false, "targets": 17 },
658-
{ "visible": false, "targets": 18 },
659-
{ "visible": false, "targets": 19 },
660-
{ "visible": false, "targets": 20 },
661-
{ "visible": false, "targets": 21 },
662-
{ "visible": false, "targets": 22 },
663-
{ "visible": false, "targets": 23 },
664-
{ "visible": false, "targets": 24 }
749+
{ "visible": false, "targets": 18 }
665750
],
666751
};
667752
var taskTableSelector = $(taskTable).DataTable(task_conf);
@@ -716,25 +801,6 @@ $(document).ready(function () {
716801
taskTableSelector.column(18).visible(true);
717802
$("#accumulator-update-table").show();
718803
}
719-
720-
if (showInputMetrics) {
721-
taskTableSelector.column(19).visible(true);
722-
}
723-
if (showOutputMetrics) {
724-
taskTableSelector.column(20).visible(true);
725-
}
726-
if (showShuffleWriteTimeMetrics) {
727-
taskTableSelector.column(21).visible(true);
728-
}
729-
if (showShuffleWriteBytesMetrics) {
730-
taskTableSelector.column(22).visible(true);
731-
}
732-
if (showMemSpill) {
733-
taskTableSelector.column(23).visible(true);
734-
}
735-
if (showDiskSpill) {
736-
taskTableSelector.column(24).visible(true);
737-
}
738804
});
739805
});
740806
});

‎core/src/main/resources/org/apache/spark/ui/static/stagespage-template.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ <h4 id="tasksTitle" class="title-table"></h4>
110110
<th>Output Size / Records</th>
111111
<th>Write Time</th>
112112
<th>Shuffle Write Size / Records</th>
113+
<th>Shuffle Read Size / Records</th>
113114
<th>Shuffle Spill (Memory)</th>
114115
<th>Shuffle Spill (Disk)</th>
115116
<th>Errors</th>

0 commit comments

Comments
 (0)