Removed changes to SparkBuild, python linalg. Fixed test failures. Renamed ParamValidate to ParamValidators. Removed explicit type from ParamValidators calls where possible.

Author: jkbradley

mllib/src/main/scala/org/apache/spark/ml/Pipeline.scala

@@ -90,6 +90,7 @@ class Pipeline extends Estimator[PipelineModel] {
     val map = extractParamMap(paramMap)
     getStages.foreach {
       case pStage: Params => pStage.validate(map)
+      case _ =>
     }
   }
 

mllib/src/main/scala/org/apache/spark/ml/feature/HashingTF.scala

@@ -19,7 +19,7 @@ package org.apache.spark.ml.feature
 
 import org.apache.spark.annotation.AlphaComponent
 import org.apache.spark.ml.UnaryTransformer
-import org.apache.spark.ml.param.{ParamValidate, IntParam, ParamMap}
+import org.apache.spark.ml.param.{ParamValidators, IntParam, ParamMap}
 import org.apache.spark.mllib.feature
 import org.apache.spark.mllib.linalg.{VectorUDT, Vector}
 import org.apache.spark.sql.types.DataType
@@ -37,7 +37,7 @@ class HashingTF extends UnaryTransformer[Iterable[_], Vector, HashingTF] {
    * @group param
    */
   val numFeatures = new IntParam(this, "numFeatures", "number of features (> 0)",
-    ParamValidate.gt[Int](0))
+    ParamValidators.gt(0))
 
   setDefault(numFeatures -> (1 << 18))
 

mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala

@@ -19,7 +19,7 @@ package org.apache.spark.ml.feature
 
 import org.apache.spark.annotation.AlphaComponent
 import org.apache.spark.ml.UnaryTransformer
-import org.apache.spark.ml.param.{ParamValidate, DoubleParam, ParamMap}
+import org.apache.spark.ml.param.{ParamValidators, DoubleParam, ParamMap}
 import org.apache.spark.mllib.feature
 import org.apache.spark.mllib.linalg.{VectorUDT, Vector}
 import org.apache.spark.sql.types.DataType
@@ -36,7 +36,7 @@ class Normalizer extends UnaryTransformer[Vector, Vector, Normalizer] {
    * (default: p = 2)
    * @group param
    */
-  val p = new DoubleParam(this, "p", "the p norm value", ParamValidate.gtEq[Double](1))
+  val p = new DoubleParam(this, "p", "the p norm value", ParamValidators.gtEq(1))
 
   setDefault(p -> 2.0)
 

mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala

@@ -21,7 +21,7 @@ import scala.collection.mutable
 
 import org.apache.spark.annotation.AlphaComponent
 import org.apache.spark.ml.UnaryTransformer
-import org.apache.spark.ml.param.{ParamValidate, IntParam, ParamMap}
+import org.apache.spark.ml.param.{ParamValidators, IntParam, ParamMap}
 import org.apache.spark.mllib.linalg._
 import org.apache.spark.sql.types.DataType
 
@@ -37,12 +37,12 @@ import org.apache.spark.sql.types.DataType
 class PolynomialExpansion extends UnaryTransformer[Vector, Vector, PolynomialExpansion] {
 
   /**
-   * The polynomial degree to expand, which should be larger than 1.
+   * The polynomial degree to expand, which should be >= 1.  A value of 1 means no expansion.
    * Default: 2
    * @group param
    */
-  val degree = new IntParam(this, "degree", "the polynomial degree to expand",
-    ParamValidate.gt[Int](2))
+  val degree = new IntParam(this, "degree", "the polynomial degree to expand (>= 1)",
+    ParamValidators.gt(1))
 
   setDefault(degree -> 2)
 

mllib/src/main/scala/org/apache/spark/ml/feature/Tokenizer.scala

@@ -56,7 +56,7 @@ class RegexTokenizer extends UnaryTransformer[String, Seq[String], RegexTokenize
    * @group param
    */
   val minTokenLength: IntParam = new IntParam(this, "minLength", "minimum token length (>= 0)",
-    ParamValidate.gtEq[Int](0))
+    ParamValidators.gtEq(0))
 
   /** @group setParam */
   def setMinTokenLength(value: Int): this.type = set(minTokenLength, value)

mllib/src/main/scala/org/apache/spark/ml/feature/VectorIndexer.scala

@@ -22,7 +22,7 @@ import org.apache.spark.ml.util.SchemaUtils
 import org.apache.spark.ml.{Estimator, Model}
 import org.apache.spark.ml.attribute.{BinaryAttribute, NumericAttribute, NominalAttribute,
   Attribute, AttributeGroup}
-import org.apache.spark.ml.param.{ParamValidate, IntParam, ParamMap, Params}
+import org.apache.spark.ml.param.{ParamValidators, IntParam, ParamMap, Params}
 import org.apache.spark.ml.param.shared._
 import org.apache.spark.mllib.linalg.{SparseVector, DenseVector, Vector, VectorUDT}
 import org.apache.spark.sql.{Row, DataFrame}
@@ -44,7 +44,7 @@ private[ml] trait VectorIndexerParams extends Params with HasInputCol with HasOu
   val maxCategories = new IntParam(this, "maxCategories",
     "Threshold for the number of values a categorical feature can take (>= 2)." +
       " If a feature is found to have > maxCategories values, then it is declared continuous.",
-    ParamValidate.gtEq[Int](2))
+    ParamValidators.gtEq(2))
 
   setDefault(maxCategories -> 20)
 

mllib/src/main/scala/org/apache/spark/ml/impl/tree/treeParams.scala

@@ -46,7 +46,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams {
   final val maxDepth: IntParam =
     new IntParam(this, "maxDepth", "Maximum depth of the tree. (>= 0)" +
       " E.g., depth 0 means 1 leaf node; depth 1 means 1 internal node + 2 leaf nodes.",
-      ParamValidate.gtEq[Int](0))
+      ParamValidators.gtEq(0))
 
   /**
    * Maximum number of bins used for discretizing continuous features and for choosing how to split
@@ -57,7 +57,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams {
    */
   final val maxBins: IntParam = new IntParam(this, "maxBins", "Max number of bins for" +
     " discretizing continuous features.  Must be >=2 and >= number of categories for any" +
-    " categorical feature.", ParamValidate.gtEq[Int](2))
+    " categorical feature.", ParamValidators.gtEq(2))
 
   /**
    * Minimum number of instances each child must have after split.
@@ -70,7 +70,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams {
   final val minInstancesPerNode: IntParam = new IntParam(this, "minInstancesPerNode", "Minimum" +
     " number of instances each child must have after split.  If a split causes the left or right" +
     " child to have fewer than minInstancesPerNode, the split will be discarded as invalid." +
-    " Should be >= 1.", ParamValidate.gtEq[Int](1))
+    " Should be >= 1.", ParamValidators.gtEq(1))
 
   /**
    * Minimum information gain for a split to be considered at a tree node.
@@ -87,7 +87,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams {
    */
   final val maxMemoryInMB: IntParam = new IntParam(this, "maxMemoryInMB",
     "Maximum memory in MB allocated to histogram aggregation.",
-    ParamValidate.gtEq[Int](0))
+    ParamValidators.gtEq(0))
 
   /**
    * If false, the algorithm will pass trees to executors to match instances with nodes.
@@ -114,7 +114,7 @@ private[ml] trait DecisionTreeParams extends PredictorParams {
     " how often to checkpoint the cached node IDs.  E.g. 10 means that the cache will get" +
     " checkpointed every 10 iterations. This is only used if cacheNodeIds is true and if the" +
     " checkpoint directory is set in the SparkContext. Must be >= 1.",
-    ParamValidate.gtEq[Int](1))
+    ParamValidators.gtEq(1))
 
   setDefault(maxDepth -> 5, maxBins -> 32, minInstancesPerNode -> 1, minInfoGain -> 0.0,
     maxMemoryInMB -> 256, cacheNodeIds -> false, checkpointInterval -> 10)
@@ -283,7 +283,7 @@ private[ml] trait TreeEnsembleParams extends DecisionTreeParams with HasSeed {
    */
   final val subsamplingRate: DoubleParam = new DoubleParam(this, "subsamplingRate",
     "Fraction of the training data used for learning each decision tree, in range (0, 1].",
-    ParamValidate.inRange[Double](0, 1, lowerInclusive = false, upperInclusive = true))
+    ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
   setDefault(subsamplingRate -> 1.0)
 
@@ -326,7 +326,7 @@ private[ml] trait RandomForestParams extends TreeEnsembleParams {
    * @group param
    */
   final val numTrees: IntParam = new IntParam(this, "numTrees", "Number of trees to train (>= 1)",
-    ParamValidate.gtEq[Int](1))
+    ParamValidators.gtEq(1))
 
   /**
    * The number of features to consider for splits at each tree node.
@@ -396,7 +396,7 @@ private[ml] trait GBTParams extends TreeEnsembleParams with HasMaxIter {
    */
   final val stepSize: DoubleParam = new DoubleParam(this, "stepSize", "Step size (a.k.a." +
     " learning rate) in interval (0, 1] for shrinking the contribution of each estimator",
-    ParamValidate.inRange[Double](0, 1, lowerInclusive = false, upperInclusive = true))
+    ParamValidators.inRange(0, 1, lowerInclusive = false, upperInclusive = true))
 
   /* TODO: Add this doc when we add this param.  SPARK-7132
    * Threshold for stopping early when runWithValidation is used.

mllib/src/main/scala/org/apache/spark/ml/param/params.scala

@@ -35,15 +35,15 @@ import org.apache.spark.ml.util.Identifiable
  * @param name param name
  * @param doc documentation
  * @param isValid optional validation method which indicates if a value is valid.
- *                See [[ParamValidate]] for factory methods for common validation functions.
+ *                See [[ParamValidators]] for factory methods for common validation functions.
  * @tparam T param value type
  */
 @AlphaComponent
 class Param[T] (val parent: Params, val name: String, val doc: String, val isValid: T => Boolean)
   extends Serializable {
 
   def this(parent: Params, name: String, doc: String) =
-    this(parent, name, doc, ParamValidate.alwaysTrue[T])
+    this(parent, name, doc, ParamValidators.alwaysTrue[T])
 
   /**
    * Assert that the given value is valid for this parameter.
@@ -94,7 +94,7 @@ class Param[T] (val parent: Params, val name: String, val doc: String, val isVal
  * Factory methods for common validation functions for [[Param.isValid]].
  * The numerical methods only support Int, Long, Float, and Double.
  */
-object ParamValidate {
+object ParamValidators {
 
   /** (private[param]) Default validation always return true */
   private[param] def alwaysTrue[T]: T => Boolean = (_: T) => true
@@ -176,37 +176,37 @@ class DoubleParam(parent: Params, name: String, doc: String, isValid: Double =>
   extends Param[Double](parent, name, doc, isValid) {
 
   def this(parent: Params, name: String, doc: String) =
-    this(parent, name, doc, ParamValidate.alwaysTrue[Double])
+    this(parent, name, doc, ParamValidators.alwaysTrue)
 
   override def w(value: Double): ParamPair[Double] = super.w(value)
 }
 
 /** Specialized version of [[Param[Int]]] for Java. */
 class IntParam(parent: Params, name: String, doc: String, isValid: Int => Boolean)
-  extends Param[Int](parent, name, doc) {
+  extends Param[Int](parent, name, doc, isValid) {
 
   def this(parent: Params, name: String, doc: String) =
-    this(parent, name, doc, ParamValidate.alwaysTrue[Int])
+    this(parent, name, doc, ParamValidators.alwaysTrue)
 
   override def w(value: Int): ParamPair[Int] = super.w(value)
 }
 
 /** Specialized version of [[Param[Float]]] for Java. */
 class FloatParam(parent: Params, name: String, doc: String, isValid: Float => Boolean)
-  extends Param[Float](parent, name, doc) {
+  extends Param[Float](parent, name, doc, isValid) {
 
   def this(parent: Params, name: String, doc: String) =
-    this(parent, name, doc, ParamValidate.alwaysTrue[Float])
+    this(parent, name, doc, ParamValidators.alwaysTrue)
 
   override def w(value: Float): ParamPair[Float] = super.w(value)
 }
 
 /** Specialized version of [[Param[Long]]] for Java. */
 class LongParam(parent: Params, name: String, doc: String, isValid: Long => Boolean)
-  extends Param[Long](parent, name, doc) {
+  extends Param[Long](parent, name, doc, isValid) {
 
   def this(parent: Params, name: String, doc: String) =
-    this(parent, name, doc, ParamValidate.alwaysTrue[Long])
+    this(parent, name, doc, ParamValidators.alwaysTrue)
 
   override def w(value: Long): ParamPair[Long] = super.w(value)
 }

mllib/src/main/scala/org/apache/spark/ml/param/shared/SharedParamsCodeGen.scala

@@ -21,7 +21,7 @@ import java.io.PrintWriter
 
 import scala.reflect.ClassTag
 
-import org.apache.spark.ml.param.ParamValidate
+import org.apache.spark.ml.param.ParamValidators
 
 /**
  * Code generator for shared params (sharedParams.scala). Run under the Spark folder with
@@ -34,9 +34,9 @@ private[shared] object SharedParamsCodeGen {
   def main(args: Array[String]): Unit = {
     val params = Seq(
       ParamDesc[Double]("regParam", "regularization parameter (>= 0)",
-        isValid = "ParamValidate.gtEq[Double](0)"),
+        isValid = "ParamValidators.gtEq(0)"),
       ParamDesc[Int]("maxIter", "max number of iterations (>= 0)",
-        isValid = "ParamValidate.gtEq[Int](0)"),
+        isValid = "ParamValidators.gtEq(0)"),
       ParamDesc[String]("featuresCol", "features column name", Some("\"features\"")),
       ParamDesc[String]("labelCol", "label column name", Some("\"label\"")),
       ParamDesc[String]("predictionCol", "prediction column name", Some("\"prediction\"")),
@@ -46,17 +46,17 @@ private[shared] object SharedParamsCodeGen {
         "column name for predicted class conditional probabilities", Some("\"probability\"")),
       ParamDesc[Double]("threshold",
         "threshold in binary classification prediction, in range [0, 1]",
-        isValid = "ParamValidate.inRange[Double](0, 1)"),
+        isValid = "ParamValidators.inRange(0, 1)"),
       ParamDesc[String]("inputCol", "input column name"),
       ParamDesc[Array[String]]("inputCols", "input column names"),
       ParamDesc[String]("outputCol", "output column name"),
       ParamDesc[Int]("checkpointInterval", "checkpoint interval (>= 1)",
-        isValid = "ParamValidate.gtEq[Int](1)"),
+        isValid = "ParamValidators.gtEq(1)"),
       ParamDesc[Boolean]("fitIntercept", "whether to fit an intercept term", Some("true")),
       ParamDesc[Long]("seed", "random seed", Some("Utils.random.nextLong()")),
       ParamDesc[Double]("elasticNetParam", "the ElasticNet mixing parameter, in range [0, 1]." +
         " For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.",
-        isValid = "ParamValidate.inRange[Double](0, 1)"),
+        isValid = "ParamValidators.inRange(0, 1)"),
       ParamDesc[Double]("tol", "the convergence tolerance for iterative algorithms"),
       ParamDesc[Double]("stepSize", "Step size to be used for each iteration of optimization."))
 

mllib/src/main/scala/org/apache/spark/ml/param/shared/sharedParams.scala

@@ -34,7 +34,7 @@ private[ml] trait HasRegParam extends Params {
    * Param for regularization parameter (>= 0).
    * @group param
    */
-  final val regParam: DoubleParam = new DoubleParam(this, "regParam", "regularization parameter (>= 0)", ParamValidate.gtEq[Double](0))
+  final val regParam: DoubleParam = new DoubleParam(this, "regParam", "regularization parameter (>= 0)", ParamValidators.gtEq(0))
 
   /** @group getParam */
   final def getRegParam: Double = getOrDefault(regParam)
@@ -49,7 +49,7 @@ private[ml] trait HasMaxIter extends Params {
    * Param for max number of iterations (>= 0).
    * @group param
    */
-  final val maxIter: IntParam = new IntParam(this, "maxIter", "max number of iterations (>= 0)", ParamValidate.gtEq[Int](0))
+  final val maxIter: IntParam = new IntParam(this, "maxIter", "max number of iterations (>= 0)", ParamValidators.gtEq(0))
 
   /** @group getParam */
   final def getMaxIter: Int = getOrDefault(maxIter)
@@ -149,7 +149,7 @@ private[ml] trait HasThreshold extends Params {
    * Param for threshold in binary classification prediction, in range [0, 1].
    * @group param
    */
-  final val threshold: DoubleParam = new DoubleParam(this, "threshold", "threshold in binary classification prediction, in range [0, 1]", ParamValidate.inRange[Double](0, 1))
+  final val threshold: DoubleParam = new DoubleParam(this, "threshold", "threshold in binary classification prediction, in range [0, 1]", ParamValidators.inRange(0, 1))
 
   /** @group getParam */
   final def getThreshold: Double = getOrDefault(threshold)
@@ -209,7 +209,7 @@ private[ml] trait HasCheckpointInterval extends Params {
    * Param for checkpoint interval (>= 1).
    * @group param
    */
-  final val checkpointInterval: IntParam = new IntParam(this, "checkpointInterval", "checkpoint interval (>= 1)", ParamValidate.gtEq[Int](1))
+  final val checkpointInterval: IntParam = new IntParam(this, "checkpointInterval", "checkpoint interval (>= 1)", ParamValidators.gtEq(1))
 
   /** @group getParam */
   final def getCheckpointInterval: Int = getOrDefault(checkpointInterval)
@@ -258,7 +258,7 @@ private[ml] trait HasElasticNetParam extends Params {
    * Param for the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty..
    * @group param
    */
-  final val elasticNetParam: DoubleParam = new DoubleParam(this, "elasticNetParam", "the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.", ParamValidate.inRange[Double](0, 1))
+  final val elasticNetParam: DoubleParam = new DoubleParam(this, "elasticNetParam", "the ElasticNet mixing parameter, in range [0, 1]. For alpha = 0, the penalty is an L2 penalty. For alpha = 1, it is an L1 penalty.", ParamValidators.inRange(0, 1))
 
   /** @group getParam */
   final def getElasticNetParam: Double = getOrDefault(elasticNetParam)