Skip to content

Commit 2a79f26

Browse files
Avoid loaing properties file frequently
1 parent e77fa81 commit 2a79f26

File tree

2 files changed

+4
-2
lines changed

2 files changed

+4
-2
lines changed

core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ object SparkSubmit {
280280
}
281281

282282
// Read from default spark properties, if any
283-
for ((k, v) <- args.getDefaultSparkProperties) {
283+
for ((k, v) <- args.defaultSparkProperties) {
284284
sysProps.getOrElseUpdate(k, v)
285285
}
286286

core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
5656
var isPython: Boolean = false
5757
var pyFiles: String = null
5858
val sparkProperties: HashMap[String, String] = new HashMap[String, String]()
59+
val defaultSparkProperties: HashMap[String, String] = new HashMap[String, String]()
5960

6061
parseOpts(args.toList)
6162
mergeSparkProperties()
@@ -108,6 +109,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
108109
}
109110

110111
val properties = getDefaultSparkProperties
112+
defaultSparkProperties.putAll(properties)
111113
properties.putAll(sparkProperties)
112114

113115
// Use properties file as fallback for values which have a direct analog to
@@ -213,7 +215,7 @@ private[spark] class SparkSubmitArguments(args: Seq[String]) {
213215
| verbose $verbose
214216
|
215217
|Default properties from $propertiesFile:
216-
|${getDefaultSparkProperties.mkString(" ", "\n ", "\n")}
218+
|${defaultSparkProperties.mkString(" ", "\n ", "\n")}
217219
""".stripMargin
218220
}
219221

0 commit comments

Comments
 (0)