[SPARK-28647][WEBUI][2.4] Recover additional metric feature

This PR is for backporting SPARK-28647(apache#25374) to branch-2.4.
The original PR removed `additional-metrics.js` but branch-2.4 still uses it so I don't remove it and related things for branch-2.4.

### What changes were proposed in this pull request?
Added checkboxes to enable users to select which optional metrics (`On Heap Memory`, `Off Heap Memory` and `Select All` in this case) to be shown in `ExecuorPage`.

### Why are the changes needed?
By SPARK-17019, `On Heap Memory` and `Off Heap Memory` are introduced as optional metrics. But they are not displayed because they are made `display: none` in css and there are no way to appear them.

### Does this PR introduce any user-facing change?
The previous `ExecutorPage` doesn't show optional metrics.
This change adds checkboxes to `ExecutorPage` for optional metrics.
We can choose which metrics should be shown by checking corresponding checkboxes.
![Screenshot from 2019-08-18 03-56-09](https://user-images.githubusercontent.com/4736016/63216148-2bfadb80-c16c-11e9-81e1-e1e66198dd6c.png)

### How was this patch tested?
Manual test.

Closes apache#25484 from sarutak/backport-SPARK-28647-branch-2.4.

Authored-by: Kousuke Saruta <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: sarutak

core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html

@@ -16,7 +16,8 @@
 -->
 
 <script id="executors-summary-template" type="text/html">
-  <h4 style="clear: left; display: inline-block;">Summary</h4>
+  <div id="showAdditionalMetrics"></div>
+  <h4 class="title-table">Summary</h4>
   <div class="container-fluid">
     <div class="container-fluid">
       <table id="summary-execs-table" class="table table-striped compact">
@@ -26,11 +27,11 @@ <h4 style="clear: left; display: inline-block;">Summary</h4>
         <th><span data-toggle="tooltip"
                   title="Memory used / total available memory for storage of data like RDD partitions cached in memory.">Storage Memory</span>
         </th>
-        <th class="on_heap_memory">
+        <th>
           <span data-toggle="tooltip"
                 title="Memory used / total available memory for on heap storage of data like RDD partitions cached in memory.">On Heap Storage Memory</span>
         </th>
-        <th class="off_heap_memory">
+        <th>
           <span data-toggle="tooltip"
                 title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">Off Heap Storage Memory</span>
         </th>
@@ -81,11 +82,11 @@ <h4 style="clear: left; display: inline-block;">Executors</h4>
             <span data-toggle="tooltip" data-placement="top"
                   title="Memory used / total available memory for storage of data like RDD partitions cached in memory.">
               Storage Memory</span></th>
-          <th class="on_heap_memory">
+          <th>
             <span data-toggle="tooltip" data-placement="top"
                   title="Memory used / total available memory for on heap storage of data like RDD partitions cached in memory.">
               On Heap Storage Memory</span></th>
-          <th class="off_heap_memory">
+          <th>
             <span data-toggle="tooltip"
                   title="Memory used / total available memory for off heap storage of data like RDD partitions cached in memory.">
               Off Heap Storage Memory</span></th>

core/src/main/resources/org/apache/spark/ui/static/executorspage.js

@@ -177,14 +177,35 @@ function totalDurationColor(totalGCTime, totalDuration) {
     return (totalGCTime > GCTimePercent * totalDuration) ? "white" : "black";
 }
 
+var sumOptionalColumns = [3, 4];
+var execOptionalColumns = [5, 6];
+var execDataTable;
+var sumDataTable;
+
+function reselectCheckboxesBasedOnTaskTableState() {
+    var allChecked = true;
+    if (typeof execDataTable !== "undefined") {
+        for (var k = 0; k < execOptionalColumns.length; k++) {
+            if (execDataTable.column(execOptionalColumns[k]).visible()) {
+                $("[data-exec-col-idx=" + execOptionalColumns[k] + "]").prop("checked", true);
+            } else {
+                allChecked = false;
+            }
+        }
+    }
+    if (allChecked) {
+        $("#select-all-box").prop("checked", true);
+    }
+}
+
 $(document).ready(function () {
     $.extend($.fn.dataTable.defaults, {
         stateSave: true,
         lengthMenu: [[20, 40, 60, 100, -1], [20, 40, 60, 100, "All"]],
         pageLength: 20
     });
 
-    executorsSummary = $("#active-executors");
+    var executorsSummary = $("#active-executors");
 
     getStandAloneppId(function (appId) {
 
@@ -444,9 +465,6 @@ $(document).ready(function () {
                                 else
                                     return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
                                         formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                $(nTd).addClass('on_heap_memory')
                             }
                         },
                         {
@@ -456,9 +474,6 @@ $(document).ready(function () {
                                 else
                                     return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
                                         formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                $(nTd).addClass('off_heap_memory')
                             }
                         },
                         {data: 'diskUsed', render: formatBytes},
@@ -505,12 +520,16 @@ $(document).ready(function () {
                             }
                         }
                     ],
-                    "order": [[0, "asc"]]
+                    "order": [[0, "asc"]],
+                    "columnDefs": [
+                        {"visible": false, "targets": 5},
+                        {"visible": false, "targets": 6}
+                    ]
                 };
-    
-                var dt = $(selector).DataTable(conf);
-                dt.column('executorLogsCol:name').visible(logsExist(response));
-                dt.column('threadDumpCol:name').visible(getThreadDumpEnabled());
+
+                execDataTable = $(selector).DataTable(conf);
+                execDataTable.column('executorLogsCol:name').visible(logsExist(response));
+                execDataTable.column('threadDumpCol:name').visible(getThreadDumpEnabled());
                 $('#active-executors [data-toggle="tooltip"]').tooltip();
 
                 var sumSelector = "#summary-execs-table";
@@ -540,9 +559,6 @@ $(document).ready(function () {
                                 else
                                     return (formatBytes(row.allOnHeapMemoryUsed, type) + ' / ' +
                                         formatBytes(row.allOnHeapMaxMemory, type));
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                $(nTd).addClass('on_heap_memory')
                             }
                         },
                         {
@@ -552,9 +568,6 @@ $(document).ready(function () {
                                 else
                                     return (formatBytes(row.allOffHeapMemoryUsed, type) + ' / ' +
                                         formatBytes(row.allOffHeapMaxMemory, type));
-                            },
-                            "fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
-                                $(nTd).addClass('off_heap_memory')
                             }
                         },
                         {data: 'allDiskUsed', render: formatBytes},
@@ -597,13 +610,69 @@ $(document).ready(function () {
                     ],
                     "paging": false,
                     "searching": false,
-                    "info": false
+                    "info": false,
+                    "columnDefs": [
+                        {"visible": false, "targets": 3},
+                        {"visible": false, "targets": 4}
+                    ]
 
                 };
 
-                $(sumSelector).DataTable(sumConf);
+                sumDataTable = $(sumSelector).DataTable(sumConf);
                 $('#execSummary [data-toggle="tooltip"]').tooltip();
-    
+
+                $("#showAdditionalMetrics").append(
+                    "<div><a id='additionalMetrics'>" +
+                    "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle-optional-metrics'></span>" +
+                    "Show Additional Metrics" +
+                    "</a></div>" +
+                    "<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>" +
+                    "<div><input type='checkbox' class='toggle-vis' id='select-all-box'>Select All</div>" +
+                    "<div id='on_heap_memory' class='on-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='3' data-exec-col-idx='5'>On Heap Memory</div>" +
+                    "<div id='off_heap_memory' class='off-heap-memory-checkbox-div'><input type='checkbox' class='toggle-vis' data-sum-col-idx='4' data-exec-col-idx='6'>Off Heap Memory</div>" +
+                    "</div>");
+
+                reselectCheckboxesBasedOnTaskTableState();
+
+                $("#additionalMetrics").click(function() {
+                    $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+                    $("#toggle-metrics").toggle();
+                    if (window.localStorage) {
+                        window.localStorage.setItem("arrowtoggle-optional-metrics-class", $("#arrowtoggle-optional-metrics").attr('class'));
+                    }
+                });
+
+                $(".toggle-vis").on("click", function() {
+                    var thisBox = $(this);
+                    if (thisBox.is("#select-all-box")) {
+                        var sumColumn = sumDataTable.columns(sumOptionalColumns);
+                        var execColumn = execDataTable.columns(execOptionalColumns);
+                        if (thisBox.is(":checked")) {
+                            $(".toggle-vis").prop("checked", true);
+                            sumColumn.visible(true);
+                            execColumn.visible(true);
+                        } else {
+                            $(".toggle-vis").prop("checked", false);
+                            sumColumn.visible(false);
+                            execColumn.visible(false);
+                        }
+                    } else {
+                        var execColIdx = thisBox.attr("data-exec-col-idx");
+                        var execCol = execDataTable.column(execColIdx);
+                        execCol.visible(!execCol.visible());
+                        var sumColIdx = thisBox.attr("data-sum-col-idx");
+                        var sumCol = sumDataTable.column(sumColIdx);
+                        sumCol.visible(!sumCol.visible());
+                    }
+                });
+
+                if (window.localStorage) {
+                    if (window.localStorage.getItem("arrowtoggle-optional-metrics-class") != null &&
+                        window.localStorage.getItem("arrowtoggle-optional-metrics-class").includes("arrow-open")) {
+                        $("#arrowtoggle-optional-metrics").toggleClass("arrow-open arrow-closed");
+                        $("#toggle-metrics").toggle();
+                    }
+                }
             });
         });
     });

core/src/main/resources/org/apache/spark/ui/static/webui.css

@@ -260,4 +260,17 @@ a.expandbutton {
 .paginate_button.active > a {
     color: #999999;
     text-decoration: underline;
-}
+}
+
+.title-table {
+  clear: left;
+  display: inline-block;
+}
+
+.table-dataTable {
+  width: 100%;
+}
+
+.container-fluid-div {
+  width: 200px;
+}