[SPARK-45120][SPARK-45150][UI] Upgrade d3 from v3 to v7(v7.8.5) and apply api changes in UI

### What changes were proposed in this pull request?

This PR is a follow-up of SPARK-45120 to cover the missing cases, and then fix test failures reported by SPARK-45150

It also brings back #42879

### Why are the changes needed?

bugfix

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

tested with the excluded CI tests bellow

```
[info] ChromeUISeleniumSuite:
Starting ChromeDriver 117.0.5938.62 (25a7172909a4cba7355365cf424d7d7eb35231f4-refs/branch-heads/5938{#1146}) on port 12624
Only local connections are allowed.
Please see https://chromedriver.chromium.org/security-considerations for suggestions on keeping ChromeDriver safe.
ChromeDriver was started successfully.
[info] - SPARK-31534: text for tooltip should be escaped (1 second, 979 milliseconds)
[info] - SPARK-31882: Link URL for Stage DAGs should not depend on paged table. (683 milliseconds)
[info] - SPARK-31886: Color barrier execution mode RDD correctly (304 milliseconds)
[info] - Search text for paged tables should not be saved (1 second, 307 milliseconds)
[info] Run completed in 6 seconds, 702 milliseconds.
[info] Total number of tests run: 4
[info] Suites: completed 1, aborted 0
[info] Tests: succeeded 4, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
````

### Was this patch authored or co-authored using generative AI tooling?

no

Closes #42907 from yaooqinn/SPARK-45150.

Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>

Author: yaooqinn

LICENSE

@@ -229,7 +229,6 @@ BSD 3-Clause
 python/lib/py4j-*-src.zip
 python/pyspark/cloudpickle/*.py
 python/pyspark/join.py
-core/src/main/resources/org/apache/spark/ui/static/d3.min.js
 
 The CSS style for the navigation sidebar of the documentation was originally
 submitted by Óscar Nájera for the scikit-learn project. The scikit-learn project
@@ -248,6 +247,11 @@ docs/js/vendor/anchor.min.js
 docs/js/vendor/jquery*
 docs/js/vendor/modernizer*
 
+ISC License
+-----------
+
+core/src/main/resources/org/apache/spark/ui/static/d3.min.js
+
 
 Creative Commons CC0 1.0 Universal Public Domain Dedication
 -----------------------------------------------------------

LICENSE-binary

@@ -461,7 +461,6 @@ org.jdom:jdom2
 python/lib/py4j-*-src.zip
 python/pyspark/cloudpickle.py
 python/pyspark/join.py
-core/src/main/resources/org/apache/spark/ui/static/d3.min.js
 
 The CSS style for the navigation sidebar of the documentation was originally
 submitted by Óscar Nájera for the scikit-learn project. The scikit-learn project
@@ -498,6 +497,11 @@ docs/js/vendor/anchor.min.js
 docs/js/vendor/jquery*
 docs/js/vendor/modernizer*
 
+ISC License
+-----------
+
+core/src/main/resources/org/apache/spark/ui/static/d3.min.js
+
 
 Common Development and Distribution License (CDDL) 1.0
 ------------------------------------------------------

core/src/main/resources/org/apache/spark/ui/static/d3.min.js

@@ -249,9 +249,9 @@ function renderDagVizForJob(svgContainer) {
     // existing ones, taking into account the position and width of the last stage's
     // container. We do not need to do this for the first stage of this job.
     if (i > 0) {
-      var existingStages = svgContainer.selectAll("g.cluster.stage");
-      if (!existingStages.empty()) {
-        var lastStage = d3.select(existingStages[0].pop());
+      var existingStages = svgContainer.selectAll("g.cluster.stage").nodes();
+      if (existingStages.length > 0) {
+        var lastStage = d3.select(existingStages.pop());
         var lastStageWidth = toFloat(lastStage.select("rect").attr("width"));
         var lastStagePosition = getAbsolutePosition(lastStage);
         var offset = lastStagePosition.x + lastStageWidth + VizConstants.stageSep;
@@ -346,7 +346,7 @@ function preprocessGraphLayout(g, forJob) {
  * This assumes that all outermost elements are clusters (rectangles).
  */
 function resizeSvg(svg) {
-  var allClusters = svg.selectAll("g.cluster rect")[0];
+  var allClusters = svg.selectAll("g.cluster rect").nodes();
   var startX = -VizConstants.svgMarginX +
     toFloat(d3.min(allClusters, function(e) {
       return getAbsolutePosition(d3.select(e)).x;
@@ -380,14 +380,12 @@ function resizeSvg(svg) {
 function interpretLineBreak(svg) {
   svg.selectAll("tspan").each(function() {
     var node = d3.select(this);
-    var original = node[0][0].innerHTML;
-    if (original.indexOf("\\n") != -1) {
+    var original = node.html();
+    if (original.indexOf("\\n") !== -1) {
       var arr = original.split("\\n");
       var newNode = this.cloneNode(this);
-
-      node[0][0].innerHTML = arr[0];
-      newNode.innerHTML = arr[1];
-
+      node.html(arr[0])
+      newNode.html(arr[1]);
       this.parentNode.appendChild(newNode);
     }
   });
@@ -433,7 +431,7 @@ function getAbsolutePosition(d3selection) {
   while (!obj.empty()) {
     var transformText = obj.attr("transform");
     if (transformText) {
-      var translate = d3.transform(transformText).translate;
+      var translate = transformText.substring("translate(".length, transformText.length - 1).split(",")
       _x += toFloat(translate[0]);
       _y += toFloat(translate[1]);
     }
@@ -497,7 +495,7 @@ function connectRDDs(fromRDDId, toRDDId, edgesContainer, svgContainer) {
     ];
   }
 
-  var line = d3.svg.line().interpolate("basis");
+  var line = d3.line().curve(d3.curveBasis);
   edgesContainer.append("path").datum(points).attr("d", line);
 }
 

core/src/main/resources/org/apache/spark/ui/static/spark-dag-viz.js

@@ -100,16 +100,15 @@ function registerTimeline(minY, maxY) {
 // Register a histogram graph. All histogram graphs should be register before calling any
 // "drawHistogram" so that we can determine the max X value for histograms.
 function registerHistogram(values, minY, maxY) {
-  var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
-  // d.x is the y values while d.y is the x values
-  var maxX = d3.max(data, function(d) { return d.y; });
+  var data = d3.bin().domain([minY, maxY]).thresholds(histogramBinCount)(values);
+  var maxX = d3.max(data, (d) => d.length);
   maxXForHistogram = maxX > maxXForHistogram ? maxX : maxXForHistogram;
 }
 /* eslint-enable no-unused-vars */
 
 // Draw a line between (x1, y1) and (x2, y2)
 function drawLine(svg, xFunc, yFunc, x1, y1, x2, y2) {
-  var line = d3.svg.line()
+  var line = d3.line()
     .x(function(d) { return xFunc(d.x); })
     .y(function(d) { return yFunc(d.y); });
   var data = [{x: x1, y: y1}, {x: x2, y: y2}];
@@ -141,10 +140,10 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
   var width = 500 - margin.left - margin.right;
   var height = 150 - margin.top - margin.bottom;
 
-  var x = d3.scale.linear().domain([minX, maxX]).range([0, width]);
-  var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+  var x = d3.scaleLinear().domain([minX, maxX]).range([0, width]);
+  var y = d3.scaleLinear().domain([minY, maxY]).range([height, 0]);
 
-  var xAxis = d3.svg.axis().scale(x).orient("bottom").tickFormat(function(d) {
+  var xAxis = d3.axisBottom(x).tickFormat(function(d) {
     var formattedDate = timeFormat[d];
     var dotIndex = formattedDate.indexOf('.');
     if (dotIndex >= 0) {
@@ -154,10 +153,9 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
       return formattedDate;
     }
   });
-  var formatYValue = d3.format(",.2f");
-  var yAxis = d3.svg.axis().scale(y).orient("left").ticks(5).tickFormat(formatYValue);
+  var yAxis = d3.axisLeft(y).ticks(5).tickFormat(yValueFormat);
 
-  var line = d3.svg.line()
+  var line = d3.line()
     .x(function(d) { return x(d.x); })
     .y(function(d) { return y(d.y); });
 
@@ -213,7 +211,7 @@ function drawTimeline(id, data, minX, maxX, minY, maxY, unitY, batchInterval) {
     .attr("cy", function(d) { return y(d.y); })
     .attr("r", function(d) { return isFailedBatch(d.x) ? "2" : "3";})
     .on('mouseover', function(d) {
-      var tip = formatYValue(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
+      var tip = yValueFormat(d.y) + " " + unitY + " at " + timeTipStrings[d.x];
       showBootstrapTooltip(d3.select(this).node(), tip);
       // show the point
       d3.select(this)
@@ -252,13 +250,19 @@ function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
   var width = 350 - margin.left - margin.right;
   var height = 150 - margin.top - margin.bottom;
 
-  var x = d3.scale.linear().domain([0, maxXForHistogram]).range([0, width - 50]);
-  var y = d3.scale.linear().domain([minY, maxY]).range([height, 0]);
+  var x = d3
+    .scaleLinear()
+    .domain([0, maxXForHistogram])
+    .range([0, width - 50]);
+  var y = d3
+    .scaleLinear()
+    .domain([minY, maxY])
+    .range([height, 0]);
 
-  var xAxis = d3.svg.axis().scale(x).orient("top").ticks(5);
-  var yAxis = d3.svg.axis().scale(y).orient("left").ticks(0).tickFormat(function(d) { return ""; });
+  var xAxis = d3.axisTop(x).ticks(5);
+  var yAxis = d3.axisLeft(y).ticks(0).tickFormat(function(d) { return ""; });
 
-  var data = d3.layout.histogram().range([minY, maxY]).bins(histogramBinCount)(values);
+  var data = d3.bin().domain([minY, maxY]).thresholds(histogramBinCount)(values);
 
   var svg = d3.select(id).append("svg")
     .attr("width", width + margin.left + margin.right)
@@ -285,13 +289,13 @@ function drawHistogram(id, values, minY, maxY, unitY, batchInterval) {
     .data(data)
     .enter()
     .append("g")
-    .attr("transform", function(d) { return "translate(0," + (y(d.x) - height + y(d.dx))  + ")";})
+    .attr("transform", (d, i) => "translate(0," + (height - (i + 1) * (y(d.x0) - y(d.x1))) + ")")
     .attr("class", "bar").append("rect")
-    .attr("width", function(d) { return x(d.y); })
-    .attr("height", function(d) { return height - y(d.dx); })
-    .on('mouseover', function(d) {
-      var percent = yValueFormat(d.y * 100.0 / values.length) + "%";
-      var tip = d.y + " batches (" + percent + ") between " + yValueFormat(d.x) + " and " + yValueFormat(d.x + d.dx) + " " + unitY;
+    .attr("width", (d) => x(d.length))
+    .attr("height", (d) => y(d.x0) - y(d.x1))
+    .on('mouseover', function(event, d) {
+      var percent = yValueFormat(d.length / values.length) + "%";
+      var tip = d.length + " batches (" + percent + ") between " + yValueFormat(d.x0) + " and " + yValueFormat(d.x1) + " " + unitY;
       showBootstrapTooltip(d3.select(this).node(), tip);
     })
     .on('mouseout',  function() {

core/src/main/resources/org/apache/spark/ui/static/streaming-page.js

@@ -39,37 +39,30 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
 
   var data = values;
 
-  var parse = d3.time.format("%H:%M:%S.%L").parse;
+  var parse = d3.timeParse("%H:%M:%S.%L");
 
   // Transpose the data into layers
-  var dataset = d3.layout.stack()(labels.map(function(fruit) {
+  var dataset = d3.stack()(labels.map(function(fruit) {
     return data.map(function(d) {
       return {_x: d.x, x: parse(d.x), y: +d[fruit]};
     });
   }));
 
   // Set x, y and colors
-  var x = d3.scale.ordinal()
+  var x = d3.scaleOrdinal()
     .domain(dataset[0].map(function(d) { return d.x; }))
     .rangeRoundBands([10, width-10], 0.02);
 
-  var y = d3.scale.linear()
+  var y = d3.scaleLinear()
     .domain([0, d3.max(dataset, function(d) {  return d3.max(d, function(d) { return d.y0 + d.y; });  })])
     .range([height, 0]);
 
   var colors = colorPool.slice(0, labels.length);
 
   // Define and draw axes
-  var yAxis = d3.svg.axis()
-    .scale(y)
-    .orient("left")
-    .ticks(7)
-    .tickFormat( function(d) { return d } );
+  var yAxis = d3.axisLeft(y).ticks(7).tickFormat( function(d) { return d } );
 
-  var xAxis = d3.svg.axis()
-    .scale(x)
-    .orient("bottom")
-    .tickFormat(d3.time.format("%H:%M:%S.%L"));
+  var xAxis = d3.axisBottom(x).tickFormat(d3.timeFormat("%H:%M:%S.%L"));
 
   // Only show the first and last time in the graph
   var xline = [];
@@ -118,9 +111,9 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
     .on('mouseout',  function() {
       hideBootstrapTooltip(d3.select(this).node());
     })
-    .on("mousemove", function(d) {
-      var xPosition = d3.mouse(this)[0] - 15;
-      var yPosition = d3.mouse(this)[1] - 25;
+    .on("mousemove", (event, d) => {
+      var xPosition = d3.pointer(event)[0] - 15;
+      var yPosition = d3.pointer(event)[1] - 25;
       tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
       tooltip.select("text").text(d.y);
     });
@@ -144,9 +137,9 @@ function drawAreaStack(id, labels, values, minX, maxX, minY, maxY) {
     .on('mouseout',  function() {
       hideBootstrapTooltip(d3.select(this).node());
     })
-    .on("mousemove", function(d) {
-      var xPosition = d3.mouse(this)[0] - 15;
-      var yPosition = d3.mouse(this)[1] - 25;
+    .on("mousemove", (event, d) => {
+      var xPosition = d3.pointer(event)[0] - 15;
+      var yPosition = d3.pointer(event)[1] - 25;
       tooltip.attr("transform", "translate(" + xPosition + "," + yPosition + ")");
       tooltip.select("text").text(d.y);
     });

core/src/main/resources/org/apache/spark/ui/static/structured-streaming-page.js

@@ -1,26 +1,13 @@
-Copyright (c) 2010-2015, Michael Bostock
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-* The name Michael Bostock may not be used to endorse or promote products
-  derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+Copyright 2010-2023 Mike Bostock
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.

licenses-binary/LICENSE-d3.min.js.txt

@@ -1,26 +1,13 @@
-Copyright (c) 2010-2015, Michael Bostock
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-* The name Michael Bostock may not be used to endorse or promote products
-  derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
-INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
-BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
-OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
-EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+Copyright 2010-2023 Mike Bostock
+
+Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.

licenses/LICENSE-d3.min.js.txt

@@ -121,7 +121,7 @@ function preprocessGraphLayout(g) {
  * This assumes that all outermost elements are clusters (rectangles).
  */
 function resizeSvg(svg) {
-  var allClusters = svg.selectAll("g rect")[0];
+  var allClusters = svg.selectAll("g rect").nodes();
   var startX = -PlanVizConstants.svgMarginX +
     toFloat(d3.min(allClusters, function(e) {
       return getAbsolutePosition(d3.select(e)).x;
@@ -169,7 +169,7 @@ function getAbsolutePosition(d3selection) {
   while (!obj.empty()) {
     var transformText = obj.attr("transform");
     if (transformText) {
-      var translate = d3.transform(transformText).translate;
+      var translate = transformText.substring("translate(".length, transformText.length - 1).split(",")
       _x += toFloat(translate[0]);
       _y += toFloat(translate[1]);
     }