File tree Expand file tree Collapse file tree 1 file changed +7
-1
lines changed
core/src/main/scala/org/apache/spark Expand file tree Collapse file tree 1 file changed +7
-1
lines changed Original file line number Diff line number Diff line change @@ -1495,14 +1495,20 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
14951495 throw new IllegalStateException (" listener bus has already been started" )
14961496 }
14971497 // Use reflection to instantiate listeners specified via the `spark.extraListeners`
1498- // configuration or the SPARK_EXTRA_LISTENERS environment variable
1498+ // configuration or the SPARK_EXTRA_LISTENERS environment variable. The purpose of
1499+ // SPARK_EXTRA_LISTENERS is to allow the execution environment to inject custom listeners
1500+ // without having to worry about them being overridden by user settings in SparkConf.
14991501 try {
15001502 val listenerClassNames : Seq [String ] = {
1503+ // Merge configurations from both sources
15011504 val fromSparkConf = conf.get(" spark.extraListeners" , " " ).split(',' )
15021505 val fromEnvVar = Option (conf.getenv(" SPARK_EXTRA_LISTENERS" )).getOrElse(" " ).split(',' )
1506+ // Filter out empty entries, which could occur when overriding environment variables
1507+ // (e.g. `export SPARK_EXTRA_LISTENERS="foo,$SPARK_EXTRA_LISTENERS`)
15031508 (fromSparkConf ++ fromEnvVar).map(_.trim).filter(_ != " " )
15041509 }
15051510 for (className <- listenerClassNames) {
1511+ // Use reflection to find the right constructor
15061512 val constructors = {
15071513 val listenerClass = Class .forName(className)
15081514 listenerClass.getConstructors.asInstanceOf [Array [Constructor [_ <: SparkListener ]]]
You can’t perform that action at this time.
0 commit comments