Skip to content

Commit 2daff9b

Browse files
committed
Add a couple of explanatory comments for SPARK_EXTRA_LISTENERS.
1 parent b9973da commit 2daff9b

File tree

1 file changed

+7
-1
lines changed

1 file changed

+7
-1
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1495,14 +1495,20 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
14951495
throw new IllegalStateException("listener bus has already been started")
14961496
}
14971497
// Use reflection to instantiate listeners specified via the `spark.extraListeners`
1498-
// configuration or the SPARK_EXTRA_LISTENERS environment variable
1498+
// configuration or the SPARK_EXTRA_LISTENERS environment variable. The purpose of
1499+
// SPARK_EXTRA_LISTENERS is to allow the execution environment to inject custom listeners
1500+
// without having to worry about them being overridden by user settings in SparkConf.
14991501
try {
15001502
val listenerClassNames: Seq[String] = {
1503+
// Merge configurations from both sources
15011504
val fromSparkConf = conf.get("spark.extraListeners", "").split(',')
15021505
val fromEnvVar = Option(conf.getenv("SPARK_EXTRA_LISTENERS")).getOrElse("").split(',')
1506+
// Filter out empty entries, which could occur when overriding environment variables
1507+
// (e.g. `export SPARK_EXTRA_LISTENERS="foo,$SPARK_EXTRA_LISTENERS`)
15031508
(fromSparkConf ++ fromEnvVar).map(_.trim).filter(_ != "")
15041509
}
15051510
for (className <- listenerClassNames) {
1511+
// Use reflection to find the right constructor
15061512
val constructors = {
15071513
val listenerClass = Class.forName(className)
15081514
listenerClass.getConstructors.asInstanceOf[Array[Constructor[_ <: SparkListener]]]

0 commit comments

Comments
 (0)