From 273a3a6006d4f044dbb23327e10bc88772ae7ab1 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 22 Jan 2023 14:46:56 -0800 Subject: [PATCH 1/7] [SPARK-42157][CORE] spark.scheduler.mode=FAIR should provide FAIR scheduler --- conf/fairscheduler-default.xml.template | 26 +++++++++++++++++++ .../spark/scheduler/SchedulableBuilder.scala | 15 ++++++++--- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 conf/fairscheduler-default.xml.template diff --git a/conf/fairscheduler-default.xml.template b/conf/fairscheduler-default.xml.template new file mode 100644 index 000000000000..6739f26fb4f8 --- /dev/null +++ b/conf/fairscheduler-default.xml.template @@ -0,0 +1,26 @@ + + + + + + + FAIR + 1 + 0 + + diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index e7c45a9faa1d..77212f96ce1c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -61,6 +61,7 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext val schedulerAllocFile = sc.conf.get(SCHEDULER_ALLOCATION_FILE) val DEFAULT_SCHEDULER_FILE = "fairscheduler.xml" + val DEFAULT_SCHEDULER_TEMPLATE_FILE = "fairscheduler-default.xml.template" val FAIR_SCHEDULER_PROPERTIES = SparkContext.SPARK_SCHEDULER_POOL val DEFAULT_POOL_NAME = "default" val MINIMUM_SHARES_PROPERTY = "minShare" @@ -86,10 +87,16 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") Some((is, DEFAULT_SCHEDULER_FILE)) } else { - logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + - s"FIFO order. To use fair scheduling, configure pools in $DEFAULT_SCHEDULER_FILE or " + - s"set ${SCHEDULER_ALLOCATION_FILE.key} to a file that contains the configuration.") - None + val is = Utils.getSparkClassLoader.getResourceAsStream(DEFAULT_SCHEDULER_TEMPLATE_FILE) + if (is != null) { + logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") + Some((is, DEFAULT_SCHEDULER_TEMPLATE_FILE)) + } else { + logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + + s"FIFO order. To use fair scheduling, configure pools in $DEFAULT_SCHEDULER_FILE " + + s"or set ${SCHEDULER_ALLOCATION_FILE.key} to a file that contains the configuration.") + None + } } } From 17db356eaf298c7b4c88b2479acefaaede0c4cba Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 22 Jan 2023 17:11:09 -0800 Subject: [PATCH 2/7] Fix info log message --- .../scala/org/apache/spark/scheduler/SchedulableBuilder.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index 77212f96ce1c..a813ebd3b3f7 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -89,7 +89,8 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext } else { val is = Utils.getSparkClassLoader.getResourceAsStream(DEFAULT_SCHEDULER_TEMPLATE_FILE) if (is != null) { - logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") + logInfo("Creating Fair Scheduler pools from default template file: " + + s"$DEFAULT_SCHEDULER_TEMPLATE_FILE.") Some((is, DEFAULT_SCHEDULER_TEMPLATE_FILE)) } else { logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + From 7fc9906f7579f2e565a0390c8556c005e267886d Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 23 Jan 2023 10:30:32 -0800 Subject: [PATCH 3/7] Revert "Fix info log message" This reverts commit 17db356eaf298c7b4c88b2479acefaaede0c4cba. --- .../scala/org/apache/spark/scheduler/SchedulableBuilder.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index a813ebd3b3f7..77212f96ce1c 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -89,8 +89,7 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext } else { val is = Utils.getSparkClassLoader.getResourceAsStream(DEFAULT_SCHEDULER_TEMPLATE_FILE) if (is != null) { - logInfo("Creating Fair Scheduler pools from default template file: " + - s"$DEFAULT_SCHEDULER_TEMPLATE_FILE.") + logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") Some((is, DEFAULT_SCHEDULER_TEMPLATE_FILE)) } else { logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + From 664e4f7c36a03889dcb070881a7c086e1045860c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 23 Jan 2023 10:30:34 -0800 Subject: [PATCH 4/7] Revert "[SPARK-42157][CORE] spark.scheduler.mode=FAIR should provide FAIR scheduler" This reverts commit 273a3a6006d4f044dbb23327e10bc88772ae7ab1. --- conf/fairscheduler-default.xml.template | 26 ------------------- .../spark/scheduler/SchedulableBuilder.scala | 15 +++-------- 2 files changed, 4 insertions(+), 37 deletions(-) delete mode 100644 conf/fairscheduler-default.xml.template diff --git a/conf/fairscheduler-default.xml.template b/conf/fairscheduler-default.xml.template deleted file mode 100644 index 6739f26fb4f8..000000000000 --- a/conf/fairscheduler-default.xml.template +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - FAIR - 1 - 0 - - diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index 77212f96ce1c..e7c45a9faa1d 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -61,7 +61,6 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext val schedulerAllocFile = sc.conf.get(SCHEDULER_ALLOCATION_FILE) val DEFAULT_SCHEDULER_FILE = "fairscheduler.xml" - val DEFAULT_SCHEDULER_TEMPLATE_FILE = "fairscheduler-default.xml.template" val FAIR_SCHEDULER_PROPERTIES = SparkContext.SPARK_SCHEDULER_POOL val DEFAULT_POOL_NAME = "default" val MINIMUM_SHARES_PROPERTY = "minShare" @@ -87,16 +86,10 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") Some((is, DEFAULT_SCHEDULER_FILE)) } else { - val is = Utils.getSparkClassLoader.getResourceAsStream(DEFAULT_SCHEDULER_TEMPLATE_FILE) - if (is != null) { - logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") - Some((is, DEFAULT_SCHEDULER_TEMPLATE_FILE)) - } else { - logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + - s"FIFO order. To use fair scheduling, configure pools in $DEFAULT_SCHEDULER_FILE " + - s"or set ${SCHEDULER_ALLOCATION_FILE.key} to a file that contains the configuration.") - None - } + logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + + s"FIFO order. To use fair scheduling, configure pools in $DEFAULT_SCHEDULER_FILE or " + + s"set ${SCHEDULER_ALLOCATION_FILE.key} to a file that contains the configuration.") + None } } From e0d22d6dba30c584c8169d488f119d1eadf6f9e5 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 23 Jan 2023 12:16:21 -0800 Subject: [PATCH 5/7] Address comments --- .../apache/spark/scheduler/SchedulableBuilder.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index e7c45a9faa1d..057fa65e5ec0 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path import org.apache.spark.SparkContext import org.apache.spark.internal.Logging -import org.apache.spark.internal.config.SCHEDULER_ALLOCATION_FILE +import org.apache.spark.internal.config.{SCHEDULER_ALLOCATION_FILE, SCHEDULER_MODE} import org.apache.spark.scheduler.SchedulingMode.SchedulingMode import org.apache.spark.util.Utils @@ -86,9 +86,11 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext logInfo(s"Creating Fair Scheduler pools from default file: $DEFAULT_SCHEDULER_FILE") Some((is, DEFAULT_SCHEDULER_FILE)) } else { - logWarning("Fair Scheduler configuration file not found so jobs will be scheduled in " + - s"FIFO order. To use fair scheduling, configure pools in $DEFAULT_SCHEDULER_FILE or " + - s"set ${SCHEDULER_ALLOCATION_FILE.key} to a file that contains the configuration.") + val schedulingMode = SchedulingMode.withName(sc.conf.get(SCHEDULER_MODE)) + rootPool.addSchedulable(new Pool( + DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) + logInfo("Created pool: %s, schedulingMode: %s, minShare: %d, weight: %d".format( + DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) None } } From 58082c9b38dac92c32af3407adeff05a59fa0ed8 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 23 Jan 2023 23:26:48 -0800 Subject: [PATCH 6/7] Update core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala Co-authored-by: Mridul Muralidharan <1591700+mridulm@users.noreply.github.com> --- .../scala/org/apache/spark/scheduler/SchedulableBuilder.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index 057fa65e5ec0..31cdc3db5c32 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -89,7 +89,7 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext val schedulingMode = SchedulingMode.withName(sc.conf.get(SCHEDULER_MODE)) rootPool.addSchedulable(new Pool( DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) - logInfo("Created pool: %s, schedulingMode: %s, minShare: %d, weight: %d".format( + logInfo("Fair scheduler configuration not found, created default pool: %s, schedulingMode: %s, minShare: %d, weight: %d".format( DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) None } From 512429715118f18e22ef92052f8a2f1698f6d8af Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Mon, 23 Jan 2023 23:34:11 -0800 Subject: [PATCH 7/7] Indentation --- .../scala/org/apache/spark/scheduler/SchedulableBuilder.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala index 31cdc3db5c32..a30744da9ee9 100644 --- a/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala +++ b/core/src/main/scala/org/apache/spark/scheduler/SchedulableBuilder.scala @@ -89,7 +89,8 @@ private[spark] class FairSchedulableBuilder(val rootPool: Pool, sc: SparkContext val schedulingMode = SchedulingMode.withName(sc.conf.get(SCHEDULER_MODE)) rootPool.addSchedulable(new Pool( DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) - logInfo("Fair scheduler configuration not found, created default pool: %s, schedulingMode: %s, minShare: %d, weight: %d".format( + logInfo("Fair scheduler configuration not found, created default pool: " + + "%s, schedulingMode: %s, minShare: %d, weight: %d".format( DEFAULT_POOL_NAME, schedulingMode, DEFAULT_MINIMUM_SHARE, DEFAULT_WEIGHT)) None }