From e8d1fbcc4af9ff7a2d1c1352553f4d2d18a66ff6 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Thu, 17 Apr 2014 20:53:43 -0700 Subject: [PATCH 1/2] Eliminate delay between binding and checking for logs --- .../org/apache/spark/deploy/history/HistoryServer.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index cf64700f9098c..b8f56234d37ec 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -98,6 +98,11 @@ class HistoryServer( def initialize() { attachPage(new HistoryPage(this)) attachHandler(createStaticHandler(STATIC_RESOURCE_DIR, "/static")) + } + + /** Bind to the HTTP server behind this web interface. */ + override def bind() { + super.bind() logCheckingThread.start() } From b2eb46e754d51a40f9d9ddfd4ab8598876f2bb3e Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Thu, 17 Apr 2014 20:54:17 -0700 Subject: [PATCH 2/2] Document SPARK_PUBLIC_DNS and SPARK_HISTORY_OPTS for the history server --- docs/monitoring.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/monitoring.md b/docs/monitoring.md index 4c91c3a5929bf..144be3daf1208 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -39,22 +39,33 @@ You can start a the history server by executing: The base logging directory must be supplied, and should contain sub-directories that each represents an application's event logs. This creates a web interface at -`http://:18080` by default. The history server depends on the following variables: +`http://:18080` by default. The history server can be configured as follows: - + + + + + + + + +
Environment VariableMeaning
SPARK_DAEMON_MEMORYMemory to allocate to the history server. (default: 512m).Memory to allocate to the history server (default: 512m).
SPARK_DAEMON_JAVA_OPTS JVM options for the history server (default: none).
SPARK_PUBLIC_DNS + The public address for the history server. If this is not set, links to application history + may use the internal address of the server, resulting in broken links (default: none). +
SPARK_HISTORY_OPTS + spark.history.* configuration options for the history server (default: none). +
-Further, the history server can be configured as follows: -
Property NameDefaultMeaning