validate memoryfraction in sparkconf

Author: wangfei

core/src/main/scala/org/apache/spark/SparkConf.scala

@@ -236,6 +236,27 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
       }
     }
 
+    // Validate memoryFraction
+    val storageMemFraction = getDouble("spark.storage.memoryFraction", 0.6)
+    val shuffleMemFraction = getDouble("spark.shuffle.memoryFraction", 0.3)
+    val shuffleSafFraction = getDouble("spark.shuffle.safetyFraction", 0.8)
+
+    if (storageMemFraction > 1 || storageMemFraction < 0) {
+      val msg = s"spark.storage.memoryFraction should be between 0 and 1 " +
+        s"(was '$storageMemFraction')."
+      throw new IllegalArgumentException(msg)
+    }
+    if (shuffleMemFraction > 1 || shuffleMemFraction < 0) {
+      val msg = s"spark.shuffle.memoryFraction should be between 0 and 1 " +
+        s"(was '$shuffleMemFraction')."
+      throw new IllegalArgumentException(msg)
+    }
+    if (shuffleSafFraction > 1 || shuffleSafFraction < 0) {
+      val msg = s"spark.shuffle.safetyFraction should be between 0 and 1 " +
+        s"(was '$shuffleSafFraction')."
+      throw new IllegalArgumentException(msg)
+    }
+
     // Check for legacy configs
     sys.env.get("SPARK_JAVA_OPTS").foreach { value =>
       val error =

core/src/main/scala/org/apache/spark/storage/BlockManager.scala

@@ -1045,9 +1045,6 @@ private[spark] object BlockManager extends Logging {
 
   def getMaxMemory(conf: SparkConf): Long = {
     val memoryFraction = conf.getDouble("spark.storage.memoryFraction", 0.6)
-    if (memoryFraction > 1 || memoryFraction < 0) {
-      throw new IllegalArgumentException("spark.storage.memoryFraction should be between 0 and 1.")
-    }
     (Runtime.getRuntime.maxMemory * memoryFraction).toLong
   }
 

core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala

@@ -76,12 +76,6 @@ class ExternalAppendOnlyMap[K, V, C](
   private val maxMemoryThreshold = {
     val memoryFraction = sparkConf.getDouble("spark.shuffle.memoryFraction", 0.3)
     val safetyFraction = sparkConf.getDouble("spark.shuffle.safetyFraction", 0.8)
-    if (memoryFraction > 1 || memoryFraction < 0) {
-      throw new IllegalArgumentException("spark.shuffle.memoryFraction should be between 0 and 1.")
-    }
-    if (safetyFraction > 1 || safetyFraction < 0) {
-      throw new IllegalArgumentException("spark.shuffle.safetyFraction should be between 0 and 1.")
-    }
     (Runtime.getRuntime.maxMemory * memoryFraction * safetyFraction).toLong
   }