Skip to content

Commit e9339e6

Browse files
dongjoon-hyunyaooqinn
authored andcommitted
[SPARK-45774][CORE][UI] Support spark.master.ui.historyServerUrl in ApplicationPage
### What changes were proposed in this pull request? This PR aims to support a new configuration `spark.master.ui.historyServerUrl` to show a new link, **Application History UI**, from `Application Page` to `Spark History Server` in `Spark Standalone Cluster`s. This is useful when `spark.eventLog.enabled=true` and `Spark History Server` exists. A user can launch `Spark Master` with `spark.master.ui.historyServerUrl` configuration to link to it. Please note that `Spark History Server` is an orthogonal service from not only `Spark Master` but also `Spark Applications`. For example, `Spark Application`s know only `spark.eventLog.dir`, not SHS info. Moreover, you can launch multiple SHS based on the same event location with load balancer. Lastly, SHS can be behind proxy layer too. So, only the users know the exposed SHS URL. ### Why are the changes needed? **Application Detail UI** Apache Spark `Master` currently shows `Application Detail UI` link for only live Spark jobs. <img width="436" alt="Screenshot 2023-11-02 at 8 30 20 PM" src="https://github.com/apache/spark/assets/9700541/28e6398c-3bb5-4f8c-9713-3a95c451759e"> **Application History UI** This PR adds `Application History UI` link for completed jobs as the best effort when `spark.ui.historyServerUrl` is given. <img width="513" alt="Screenshot 2023-11-02 at 8 38 37 PM" src="https://github.com/apache/spark/assets/9700541/ca8b5436-f726-4b79-97c3-416cc59ea01b"> ### Does this PR introduce _any_ user-facing change? No. This is disabled by default. ### How was this patch tested? Pass the CIs with the newly added test suite. Also, manual procedure ``` $ SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=$HOME/data/history" sbin/start-history-server.sh $ SPARK_MASTER_OPTS=-Dspark.master.ui.historyServerUrl=http://localhost:18080 sbin/start-master.sh $ sbin/start-worker.sh spark://localhost:7077 $ bin/spark-shell --master spark://127.0.0.1:7077 -c spark.eventLog.enabled=true -c spark.eventLog.dir=/Users/dongjoon/data/history ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43643 from dongjoon-hyun/SPARK-45774. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Kent Yao <[email protected]>
1 parent 1359c13 commit e9339e6

File tree

4 files changed

+87
-0
lines changed

4 files changed

+87
-0
lines changed

core/src/main/scala/org/apache/spark/deploy/master/Master.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ private[deploy] class Master(
121121
if (defaultCores < 1) {
122122
throw new SparkException(s"${DEFAULT_CORES.key} must be positive")
123123
}
124+
val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL)
124125

125126
// Alternative application submission gateway that is stable across Spark versions
126127
private val restServerEnabled = conf.get(MASTER_REST_SERVER_ENABLED)

core/src/main/scala/org/apache/spark/deploy/master/ui/ApplicationPage.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ private[ui] class ApplicationPage(parent: MasterWebUI) extends WebUIPage("app")
9898
<a href={UIUtils.makeHref(parent.master.reverseProxy,
9999
app.id, app.desc.appUiUrl)}>Application Detail UI</a>
100100
</strong></li>
101+
} else if (parent.master.historyServerUrl.nonEmpty) {
102+
<li><strong>
103+
<a href={s"${parent.master.historyServerUrl.get}/history/${app.id}"}>
104+
Application History UI</a>
105+
</strong></li>
101106
}
102107
}
103108
</ul>

core/src/main/scala/org/apache/spark/internal/config/package.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1837,6 +1837,14 @@ package object config {
18371837
.intConf
18381838
.createWithDefault(8080)
18391839

1840+
private[spark] val MASTER_UI_HISTORY_SERVER_URL =
1841+
ConfigBuilder("spark.master.ui.historyServerUrl")
1842+
.doc("The URL where Spark history server is running. Please note that this assumes " +
1843+
"that all Spark jobs share the same event log location where the history server accesses.")
1844+
.version("4.0.0")
1845+
.stringConf
1846+
.createOptional
1847+
18401848
private[spark] val IO_COMPRESSION_SNAPPY_BLOCKSIZE =
18411849
ConfigBuilder("spark.io.compression.snappy.blockSize")
18421850
.doc("Block size in bytes used in Snappy compression, in the case when " +
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.deploy.master.ui
19+
20+
import java.util.Date
21+
import javax.servlet.http.HttpServletRequest
22+
23+
import org.mockito.Mockito.{mock, when}
24+
25+
import org.apache.spark.SparkFunSuite
26+
import org.apache.spark.deploy.ApplicationDescription
27+
import org.apache.spark.deploy.DeployMessages.{MasterStateResponse, RequestMasterState}
28+
import org.apache.spark.deploy.master.{ApplicationInfo, ApplicationState, Master}
29+
import org.apache.spark.resource.ResourceProfile
30+
import org.apache.spark.rpc.RpcEndpointRef
31+
32+
class ApplicationPageSuite extends SparkFunSuite {
33+
34+
private val master = mock(classOf[Master])
35+
when(master.historyServerUrl).thenReturn(Some("http://my-history-server:18080"))
36+
37+
private val rp = new ResourceProfile(Map.empty, Map.empty)
38+
private val desc = ApplicationDescription("name", Some(4), null, "appUiUrl", rp)
39+
private val appFinished = new ApplicationInfo(0, "app-finished", desc, new Date, null, 1)
40+
appFinished.markFinished(ApplicationState.FINISHED)
41+
private val appLive = new ApplicationInfo(0, "app-live", desc, new Date, null, 1)
42+
43+
private val state = mock(classOf[MasterStateResponse])
44+
when(state.completedApps).thenReturn(Array(appFinished))
45+
when(state.activeApps).thenReturn(Array(appLive))
46+
47+
private val rpc = mock(classOf[RpcEndpointRef])
48+
when(rpc.askSync[MasterStateResponse](RequestMasterState)).thenReturn(state)
49+
50+
private val masterWebUI = mock(classOf[MasterWebUI])
51+
when(masterWebUI.master).thenReturn(master)
52+
when(masterWebUI.masterEndpointRef).thenReturn(rpc)
53+
54+
test("SPARK-45774: Application Detail UI") {
55+
val request = mock(classOf[HttpServletRequest])
56+
when(request.getParameter("appId")).thenReturn("app-live")
57+
58+
val result = new ApplicationPage(masterWebUI).render(request).toString()
59+
assert(result.contains("Application Detail UI"))
60+
assert(!result.contains("Application History UI"))
61+
assert(!result.contains(master.historyServerUrl.get))
62+
}
63+
64+
test("SPARK-45774: Application History UI") {
65+
val request = mock(classOf[HttpServletRequest])
66+
when(request.getParameter("appId")).thenReturn("app-finished")
67+
68+
val result = new ApplicationPage(masterWebUI).render(request).toString()
69+
assert(!result.contains("Application Detail UI"))
70+
assert(result.contains("Application History UI"))
71+
assert(result.contains(master.historyServerUrl.get))
72+
}
73+
}

0 commit comments

Comments
 (0)