Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.

Allow submitting to the driver through an Ingress instead of NodePort #116

Closed
wants to merge 8 commits into from
Closed

Allow submitting to the driver through an Ingress instead of NodePort #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5971552
Allow submitting with an Ingress
mccheah Feb 14, 2017
7b2d8e2
Merge remote-tracking branch 'apache-spark-on-k8s/k8s-support-alterna…
mccheah Feb 15, 2017
28f5ec7
Fix docs
mccheah Feb 16, 2017
4fec8ac
Merge branch 'k8s-support-alternate-incremental' into ingress-support
mccheah Feb 16, 2017
a5100e7
Merge remote-tracking branch 'apache-spark-on-k8s/k8s-support-alterna…
mccheah Feb 16, 2017
bebeee9
Fix compiler error
mccheah Feb 16, 2017
4332fa2
Fix test compiler error
mccheah Feb 16, 2017
c2251ee
Fix test
mccheah Feb 17, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ class HistoryServer(
conf: SparkConf,
provider: ApplicationHistoryProvider,
securityManager: SecurityManager,
port: Int)
port: Int,
basePath: String = "")
extends WebUI(securityManager, securityManager.getSSLOptions("historyServer"), port, conf)
with Logging with UIRoot with ApplicationCacheOperations {

Expand Down Expand Up @@ -279,6 +280,7 @@ object HistoryServer extends Logging {
.asInstanceOf[ApplicationHistoryProvider]

val port = conf.getInt("spark.history.ui.port", 18080)
val basePath = conf.get("spark.history.ui.basePath", "")

val server = new HistoryServer(conf, provider, securityManager, port)
server.bind()
Expand Down
4 changes: 2 additions & 2 deletions core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,9 @@ private[v1] class ApiRootResource extends UIRootFromServletContext {

private[spark] object ApiRootResource {

def getServletHandler(uiRoot: UIRoot): ServletContextHandler = {
def getServletHandler(uiRoot: UIRoot, basePath: String = ""): ServletContextHandler = {
val jerseyContext = new ServletContextHandler(ServletContextHandler.NO_SESSIONS)
jerseyContext.setContextPath("/api")
jerseyContext.setContextPath(s"${basePath.stripSuffix("/")}/api")
val holder: ServletHolder = new ServletHolder(classOf[ServletContainer])
holder.setInitParameter(ServerProperties.PROVIDER_PACKAGES, "org.apache.spark.status.api.v1")
UIRootFromServletContext.setUiRoot(jerseyContext, uiRoot)
Expand Down
10 changes: 7 additions & 3 deletions core/src/main/scala/org/apache/spark/ui/SparkUI.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ private[spark] class SparkUI private (
val operationGraphListener: RDDOperationGraphListener,
var appName: String,
val basePath: String,
val apiRootResourceBasePath: String,
val startTime: Long)
extends WebUI(securityManager, securityManager.getSSLOptions("ui"), SparkUI.getUIPort(conf),
conf, basePath, "SparkUI")
Expand All @@ -73,7 +74,7 @@ private[spark] class SparkUI private (
attachTab(new ExecutorsTab(this))
attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath))
attachHandler(ApiRootResource.getServletHandler(this))
attachHandler(ApiRootResource.getServletHandler(this, apiRootResourceBasePath))
// These should be POST only, but, the YARN AM proxy won't proxy POSTs
attachHandler(createRedirectHandler(
"/jobs/job/kill", "/jobs/", jobsTab.handleKillRequest, httpMethods = Set("GET", "POST")))
Expand Down Expand Up @@ -160,8 +161,10 @@ private[spark] object SparkUI {
securityManager: SecurityManager,
appName: String,
startTime: Long): SparkUI = {
val basePath = conf.get("spark.ui.basePath", "")
create(Some(sc), conf, listenerBus, securityManager, appName,
jobProgressListener = Some(jobProgressListener), startTime = startTime)
jobProgressListener = Some(jobProgressListener), startTime = startTime,
basePath = basePath, apiRootResourceBasePath = basePath)
}

def createHistoryUI(
Expand Down Expand Up @@ -197,6 +200,7 @@ private[spark] object SparkUI {
securityManager: SecurityManager,
appName: String,
basePath: String = "",
apiRootResourceBasePath: String = "",
jobProgressListener: Option[JobProgressListener] = None,
startTime: Long): SparkUI = {

Expand All @@ -220,6 +224,6 @@ private[spark] object SparkUI {

new SparkUI(sc, conf, securityManager, environmentListener, storageStatusListener,
executorsListener, _jobProgressListener, storageListener, operationGraphListener,
appName, basePath, startTime)
appName, basePath, apiRootResourceBasePath, startTime)
}
}
16 changes: 16 additions & 0 deletions docs/running-on-kubernetes.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,22 @@ from the other deployment modes. See the [configuration page](configuration.html
for bookkeeping purposes.
</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.exposeIngress</code></td>
<td>false</td>
<td>
When initially contacting the driver, use an Ingress when the submitting client passes application dependencies to
the driver pod. The Ingress also remains and exposes the Spark UI. This requires an Ingress controller to be
installed on the cluster.
</td>
</tr>
<tr>
<td><code>spark.kubernetes.driver.ingressBasePath</code></td>
<td>(none)</td>
<td>
Base path for the ingress created for the driver. Must be provided if spark.kubernetes.driver.exposeIngress is true.
Copy link

@lins05 lins05 Feb 17, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we give an example here? Otherwise the name ingressBasePath would make people tend to think it's only a path and not something like [host]:[port]

</td>
</tr>
<tr>
<td><code>spark.kubernetes.driverSubmitTimeout</code></td>
<td>60s</td>
Expand Down
Loading