merged with master. enforcing Params.validate

Author: jkbradley

mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala

@@ -34,6 +34,16 @@ private[classification] trait LogisticRegressionParams extends ProbabilisticClas
   with HasRegParam with HasMaxIter with HasFitIntercept with HasThreshold {
 
   setDefault(regParam -> 0.1, maxIter -> 100, threshold -> 0.5)
+
+  override def validate(paramMap: ParamMap): Unit = {
+    require(getOrDefault(regParam) >= 0,
+      s"LogisticRegression regParam must be >= 0, but was ${getOrDefault(regParam)}")
+    require(getOrDefault(maxIter) >= 0,
+      s"LogisticRegression maxIter must be >= 0, but was ${getOrDefault(maxIter)}")
+    val threshold_ = getOrDefault(threshold)
+    require(threshold_ >= 0 && threshold_ <= 1,
+      s"LogisticRegression threshold must be in range [0,1], but was $threshold_")
+  }
 }
 
 /**

mllib/src/main/scala/org/apache/spark/ml/evaluation/BinaryClassificationEvaluator.scala

@@ -36,6 +36,8 @@ import org.apache.spark.sql.types.DoubleType
 class BinaryClassificationEvaluator extends Evaluator with Params
   with HasRawPredictionCol with HasLabelCol {
 
+  override def validate(paramMap: ParamMap): Unit = { }
+
   /**
    * param for metric name in evaluation
    * @group param

mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala

@@ -31,20 +31,26 @@ import org.apache.spark.sql.types.DataType
 @AlphaComponent
 class HashingTF extends UnaryTransformer[Iterable[_], Vector, HashingTF] {
 
+  override def validate(paramMap: ParamMap): Unit = {
+    require(getOrDefault(numFeatures) > 0,
+      s"HashingTF numFeatures must be > 0, but was ${getOrDefault(numFeatures)}")
+  }
+
   /**
-   * number of features
+   * Number of features.  Should be > 0.
+   * (default = 2^18^)
    * @group param
    */
-  val numFeatures = new IntParam(this, "numFeatures", "number of features")
+  val numFeatures = new IntParam(this, "numFeatures", "number of features (> 0)")
+
+  setDefault(numFeatures -> (1 << 18))
 
   /** @group getParam */
   def getNumFeatures: Int = getOrDefault(numFeatures)
 
   /** @group setParam */
   def setNumFeatures(value: Int): this.type = set(numFeatures, value)
 
-  setDefault(numFeatures -> (1 << 18))
-
   override protected def createTransformFunc(paramMap: ParamMap): Iterable[_] => Vector = {
     val hashingTF = new feature.HashingTF(paramMap(numFeatures))
     hashingTF.transform

mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala

@@ -31,20 +31,25 @@ import org.apache.spark.sql.types.DataType
 @AlphaComponent
 class Normalizer extends UnaryTransformer[Vector, Vector, Normalizer] {
 
+  override def validate(paramMap: ParamMap): Unit = {
+    require(getOrDefault(p) >= 0, s"Normalizer p must be >= 0, but was ${getOrDefault(p)}")
+  }
+
   /**
-   * Normalization in L^p^ space, p = 2 by default.
+   * Normalization in L^p^ space.  Must be >= 1.
+   * (default: p = 2)
    * @group param
    */
   val p = new DoubleParam(this, "p", "the p norm value")
 
+  setDefault(p -> 2.0)
+
   /** @group getParam */
   def getP: Double = getOrDefault(p)
 
   /** @group setParam */
   def setP(value: Double): this.type = set(p, value)
 
-  setDefault(p -> 2.0)
-
   override protected def createTransformFunc(paramMap: ParamMap): Vector => Vector = {
     val normalizer = new feature.Normalizer(paramMap(p))
     normalizer.transform

mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala

@@ -31,7 +31,7 @@ import org.apache.spark.sql.types.{StructField, StructType}
  * Params for [[StandardScaler]] and [[StandardScalerModel]].
  */
 private[feature] trait StandardScalerParams extends Params with HasInputCol with HasOutputCol {
-  
+
   /**
    * False by default. Centers the data with mean before scaling. 
    * It will build a dense output, so this does not work on sparse input 
@@ -45,6 +45,8 @@ private[feature] trait StandardScalerParams extends Params with HasInputCol with
    * @group param
    */
   val withStd: BooleanParam = new BooleanParam(this, "withStd", "Scale to unit standard deviation")
+
+  override def validate(paramMap: ParamMap): Unit = { }
 }
 
 /**
@@ -56,7 +58,7 @@ private[feature] trait StandardScalerParams extends Params with HasInputCol with
 class StandardScaler extends Estimator[StandardScalerModel] with StandardScalerParams {
 
   setDefault(withMean -> false, withStd -> true)
-  
+
   /** @group setParam */
   def setInputCol(value: String): this.type = set(inputCol, value)
 

mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala

@@ -33,6 +33,8 @@ import org.apache.spark.util.collection.OpenHashMap
  */
 private[feature] trait StringIndexerBase extends Params with HasInputCol with HasOutputCol {
 
+  override def validate(paramMap: ParamMap): Unit = { }
+
   /** Validates and transforms the input schema. */
   protected def validateAndTransformSchema(schema: StructType, paramMap: ParamMap): StructType = {
     val map = extractParamMap(paramMap)

mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala

@@ -29,6 +29,8 @@ import org.apache.spark.sql.types.{DataType, StringType, ArrayType}
 @AlphaComponent
 class Tokenizer extends UnaryTransformer[String, Seq[String], Tokenizer] {
 
+  override def validate(paramMap: ParamMap): Unit = { }
+
   override protected def createTransformFunc(paramMap: ParamMap): String => Seq[String] = {
     _.toLowerCase.split("\\s")
   }
@@ -43,20 +45,24 @@ class Tokenizer extends UnaryTransformer[String, Seq[String], Tokenizer] {
 /**
  * :: AlphaComponent ::
  * A regex based tokenizer that extracts tokens either by repeatedly matching the regex(default) 
- * or using it to split the text (set matching to false). Optional parameters also allow to fold
- * the text to lowercase prior to it being tokenized and to filer tokens using a minimal length. 
+ * or using it to split the text (set matching to false). Optional parameters also allow filtering
+ * tokens using a minimal length.
  * It returns an array of strings that can be empty.
- * The default parameters are regex = "\\p{L}+|[^\\p{L}\\s]+", matching = true, 
- * lowercase = false, minTokenLength = 1
  */
 @AlphaComponent
 class RegexTokenizer extends UnaryTransformer[String, Seq[String], RegexTokenizer] {
 
+  override def validate(paramMap: ParamMap): Unit = {
+    require(getOrDefault(minTokenLength) >= 0,
+      s"RegexTokenizer minTokenLength must be >= 0, but was ${getOrDefault(minTokenLength)}")
+  }
+
   /**
-   * param for minimum token length, default is one to avoid returning empty strings
+   * Minimum token length, >= 0.
+   * Default: 1, to avoid returning empty strings
    * @group param
    */
-  val minTokenLength: IntParam = new IntParam(this, "minLength", "minimum token length")
+  val minTokenLength: IntParam = new IntParam(this, "minLength", "minimum token length (>= 0)")
 
   /** @group setParam */
   def setMinTokenLength(value: Int): this.type = set(minTokenLength, value)
@@ -65,7 +71,8 @@ class RegexTokenizer extends UnaryTransformer[String, Seq[String], RegexTokenize
   def getMinTokenLength: Int = getOrDefault(minTokenLength)
 
   /**
-   * param sets regex as splitting on gaps (true) or matching tokens (false)
+   * Indicates whether regex splits on gaps (true) or matching tokens (false).
+   * Default: false
    * @group param
    */
   val gaps: BooleanParam = new BooleanParam(this, "gaps", "Set regex to match gaps or tokens")
@@ -77,7 +84,8 @@ class RegexTokenizer extends UnaryTransformer[String, Seq[String], RegexTokenize
   def getGaps: Boolean = getOrDefault(gaps)
 
   /**
-   * param sets regex pattern used by tokenizer 
+   * Regex pattern used by tokenizer.
+   * Default: "\\p{L}+|[^\\p{L}\\s]+"
    * @group param
    */
   val pattern: Param[String] = new Param(this, "pattern", "regex pattern used for tokenizing")

mllib/src/main/scala/org/apache/spark/ml/param/params.scala

@@ -132,12 +132,18 @@ trait Params extends Identifiable with Serializable {
   /**
    * Validates parameter values stored internally plus the input parameter map.
    * Raises an exception if any parameter is invalid.
+   *
+   * This generally checks parameters which do not specify input/output columns;
+   * input/output columns are checked during schema validation.
    */
-  def validate(paramMap: ParamMap): Unit = {}
+  def validate(paramMap: ParamMap): Unit
 
   /**
    * Validates parameter values stored internally.
    * Raise an exception if any parameter value is invalid.
+   *
+   * This generally checks parameters which do not specify input/output columns;
+   * input/output columns are checked during schema validation.
    */
   def validate(): Unit = validate(ParamMap.empty)
 

mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala

@@ -33,10 +33,11 @@ import org.apache.spark.util.Utils
 trait HasRegParam extends Params {
 
   /**
-   * Param for regularization parameter.
+   * Param for regularization parameter.  Should be >= 0.
    * @group param
    */
-  final val regParam: DoubleParam = new DoubleParam(this, "regParam", "regularization parameter")
+  final val regParam: DoubleParam =
+    new DoubleParam(this, "regParam", "regularization parameter (>= 0)")
 
   /** @group getParam */
   final def getRegParam: Double = getOrDefault(regParam)
@@ -50,10 +51,10 @@ trait HasRegParam extends Params {
 trait HasMaxIter extends Params {
 
   /**
-   * Param for max number of iterations.
+   * Param for max number of iterations.  Should be >= 0.
    * @group param
    */
-  final val maxIter: IntParam = new IntParam(this, "maxIter", "max number of iterations")
+  final val maxIter: IntParam = new IntParam(this, "maxIter", "max number of iterations (>= 0)")
 
   /** @group getParam */
   final def getMaxIter: Int = getOrDefault(maxIter)
@@ -165,7 +166,7 @@ trait HasThreshold extends Params {
    * Param for threshold in binary classification prediction.
    * @group param
    */
-  final val threshold: DoubleParam = new DoubleParam(this, "threshold", "threshold in binary classification prediction")
+  final val threshold: DoubleParam = new DoubleParam(this, "threshold", "threshold in binary classification prediction, in range [0, 1]")
 
   /** @group getParam */
   final def getThreshold: Double = getOrDefault(threshold)
@@ -233,7 +234,7 @@ trait HasCheckpointInterval extends Params {
    * Param for checkpoint interval.
    * @group param
    */
-  final val checkpointInterval: IntParam = new IntParam(this, "checkpointInterval", "checkpoint interval")
+  final val checkpointInterval: IntParam = new IntParam(this, "checkpointInterval", "checkpoint interval (>= 1)")
 
   /** @group getParam */
   final def getCheckpointInterval: Int = getOrDefault(checkpointInterval)

mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala

@@ -138,6 +138,15 @@ private[recommendation] trait ALSParams extends Params with HasMaxIter with HasR
     implicitPrefs -> false, alpha -> 1.0, userCol -> "user", itemCol -> "item",
     ratingCol -> "rating", nonnegative -> false)
 
+  override def validate(paramMap: ParamMap): Unit = {
+    require(getOrDefault(regParam) >= 0,
+      s"ALS regParam must be >= 0, but was ${getOrDefault(regParam)}")
+    require(getOrDefault(maxIter) >= 0,
+      s"ALS maxIter must be >= 0, but was ${getOrDefault(maxIter)}")
+    require(getOrDefault(checkpointInterval) >= 1,
+      s"ALS checkpointInterval must be >= 1, but was ${getOrDefault(checkpointInterval)}")
+  }
+
   /**
    * Validates and transforms the input schema.
    * @param schema input schema