leaving only two configs

Author: witgo

yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/ExecutorLauncher.scala

@@ -99,7 +99,8 @@ class ExecutorLauncher(args: ApplicationMasterArguments, conf: Configuration, sp
     val minimumMemory = appMasterResponse.getMinimumResourceCapability().getMemory()
 
     if (minimumMemory > 0) {
-      val mem = args.executorMemory + sparkConf.getInt("spark.yarn.container.memoryOverhead", 384)
+      val mem = args.executorMemory + sparkConf.getInt("spark.yarn.executor.memoryOverhead",
+        YarnAllocationHandler.MEMORY_OVERHEAD)
       val numCore = (mem  / minimumMemory) + (if (0 != (mem % minimumMemory)) 1 else 0)
 
       if (numCore > 0) {

yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala

@@ -89,16 +89,8 @@ private[yarn] class YarnAllocationHandler(
   private val pendingReleaseContainers = new ConcurrentHashMap[ContainerId, Boolean]
 
   // Additional memory overhead - in mb.
-  private def memoryOverhead: Int = {
-    var defaultMemoryOverhead = YarnAllocationHandler.MEMORY_OVERHEAD
-    sparkConf.getOption("spark.yarn.container.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    sparkConf.getOption("spark.yarn.executor.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    defaultMemoryOverhead
-  }
+  private def memoryOverhead: Int = sparkConf.getInt("spark.yarn.executor.memoryOverhead",
+    YarnAllocationHandler.MEMORY_OVERHEAD)
 
   private val numExecutorsRunning = new AtomicInteger()
   // Used to generate a unique id per executor

yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala

@@ -66,16 +66,8 @@ trait ClientBase extends Logging {
     FsPermission.createImmutable(Integer.parseInt("644", 8).toShort)
 
   // Additional memory overhead - in mb.
-  protected def memoryOverhead: Int = {
-    var defaultMemoryOverhead = YarnAllocationHandler.MEMORY_OVERHEAD
-    sparkConf.getOption("spark.yarn.container.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    sparkConf.getOption("spark.yarn.driver.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    defaultMemoryOverhead
-  }
+  protected def memoryOverhead: Int = sparkConf.getInt("spark.yarn.driver.memoryOverhead",
+    YarnAllocationHandler.MEMORY_OVERHEAD)
 
   // TODO(harvey): This could just go in ClientArguments.
   def validateArgs() = {

yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala

@@ -91,19 +91,8 @@ private[yarn] class YarnAllocationHandler(
   private val pendingReleaseContainers = new ConcurrentHashMap[ContainerId, Boolean]
 
   // Additional memory overhead - in mb.
-  private def memoryOverhead: Int = {
-    // TODO: Here we can dynamically calculate the default value.
-    // eg: val defaultMemoryOverhead = (executorMemory * 0.25D).ceil.toInt
-    var defaultMemoryOverhead = YarnAllocationHandler.MEMORY_OVERHEAD
-    sparkConf.getOption("spark.yarn.container.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    sparkConf.getOption("spark.yarn.executor.memoryOverhead").foreach { s =>
-      defaultMemoryOverhead = s.toInt
-    }
-    defaultMemoryOverhead
-  }
-
+  private def memoryOverhead: Int = sparkConf.getInt("spark.yarn.executor.memoryOverhead",
+    YarnAllocationHandler.MEMORY_OVERHEAD)
 
   // Number of container requests that have been sent to, but not yet allocated by the
   // ApplicationMaster.