[SPARK-5207] [MLLIB] [WIP] make withMean and withStd public, add constructor which uses defaults, un-refactor test class

Author: ogeagla

mllib/src/main/scala/org/apache/spark/mllib/feature/StandardScaler.scala

@@ -70,11 +70,15 @@ class StandardScaler(withMean: Boolean, withStd: Boolean) extends Logging {
 class StandardScalerModel (
     val mean: Vector,
     val variance: Vector,
-    private var withMean: Boolean = false,
-    private var withStd: Boolean = true) extends VectorTransformer {
+    var withMean: Boolean,
+    var withStd: Boolean) extends VectorTransformer {
 
   require(mean.size == variance.size)
 
+  def this(mean: Vector, variance: Vector) {
+    this(mean, variance, false, true)
+  }
+
   def setWithMean(withMean: Boolean): this.type = {
     this.withMean = withMean
     this

mllib/src/test/scala/org/apache/spark/mllib/feature/StandardScalerSuite.scala

@@ -54,45 +54,37 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     Vectors.dense(0.0, 1.9, 0.0)
   )
 
-  def validateConstant(data1: Array[Vector], data2: Array[Vector], data3: Array[Vector]) {
-    assert(data1.forall(_.toArray.forall(_ == 0.0)),
-      "The variance is zero, so the transformed result should be 0.0")
-    assert(data2.forall(_.toArray.forall(_ == 0.0)),
-      "The variance is zero, so the transformed result should be 0.0")
-    assert(data3.forall(_.toArray.forall(_ == 0.0)),
-      "The variance is zero, so the transformed result should be 0.0")
+  private def computeSummary(data: RDD[Vector]): MultivariateStatisticalSummary = {
+    data.treeAggregate(new MultivariateOnlineSummarizer)(
+      (aggregator, data) => aggregator.add(data),
+      (aggregator1, aggregator2) => aggregator1.merge(aggregator2))
   }
 
-  def validateSparse(data: Array[Vector], dataRDD: RDD[Vector]) = {
+  test("Standardization with dense input when means and variances are provided") {
 
-    val summary = computeSummary(dataRDD)
+    val dataRDD = sc.parallelize(denseData, 3)
 
-    assert((sparseData, data, dataRDD.collect()).zipped.forall {
-      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
-      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
-      case _ => false
-    }, "The vector type should be preserved after standardization.")
+    val standardizer1 = new StandardScaler(withMean = true, withStd = true)
+    val standardizer2 = new StandardScaler()
+    val standardizer3 = new StandardScaler(withMean = true, withStd = false)
 
-    assert((data, dataRDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
+    val model1 = standardizer1.fit(dataRDD)
+    val model2 = standardizer2.fit(dataRDD)
+    val model3 = standardizer3.fit(dataRDD)
 
-    assert(summary.mean !~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
-    assert(summary.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
+    val equivalentModel1 = new StandardScalerModel(model1.mean, model1.variance, true, true)
+    val equivalentModel2 = new StandardScalerModel(model2.mean, model2.variance)
+    val equivalentModel3 = new StandardScalerModel(model3.mean, model3.variance, true, false)
 
-    assert(data(4) ~== Vectors.sparse(3, Seq((0, 0.865538862), (1, -0.22604255))) absTol 1E-5)
-    assert(data(5) ~== Vectors.sparse(3, Seq((1, 0.71580142))) absTol 1E-5)
-  }
+    val data1 = denseData.map(equivalentModel1.transform)
+    val data2 = denseData.map(equivalentModel2.transform)
+    val data3 = denseData.map(equivalentModel3.transform)
 
-  def validateDense(
-      data1: Array[Vector],
-      data2: Array[Vector],
-      data3: Array[Vector],
-      refDataRDD: RDD[Vector],
-      data1RDD: RDD[Vector],
-      data2RDD: RDD[Vector],
-      data3RDD: RDD[Vector]
-      ) = {
-
-    val refSummary = computeSummary(refDataRDD)
+    val data1RDD = equivalentModel1.transform(dataRDD)
+    val data2RDD = equivalentModel2.transform(dataRDD)
+    val data3RDD = equivalentModel3.transform(dataRDD)
+
+    val summary = computeSummary(dataRDD)
     val summary1 = computeSummary(data1RDD)
     val summary2 = computeSummary(data2RDD)
     val summary3 = computeSummary(data3RDD)
@@ -126,7 +118,7 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     assert(summary2.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
 
     assert(summary3.mean ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
-    assert(summary3.variance ~== refSummary.variance absTol 1E-5)
+    assert(summary3.variance ~== summary.variance absTol 1E-5)
 
     assert(data1(0) ~== Vectors.dense(-1.31527964, 1.023470449, 0.11637768424) absTol 1E-5)
     assert(data1(3) ~== Vectors.dense(1.637735298, 0.156973995, 1.32247368462) absTol 1E-5)
@@ -136,13 +128,7 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     assert(data3(5) ~== Vectors.dense(-0.58333333, 2.316666666, 0.18333333333) absTol 1E-5)
   }
 
-  private def computeSummary(data: RDD[Vector]): MultivariateStatisticalSummary = {
-    data.treeAggregate(new MultivariateOnlineSummarizer)(
-      (aggregator, data) => aggregator.add(data),
-      (aggregator1, aggregator2) => aggregator1.merge(aggregator2))
-  }
-
-  test("Standardization with dense input when means and variances are provided") {
+  test("Standardization with dense input") {
 
     val dataRDD = sc.parallelize(denseData, 3)
 
@@ -154,42 +140,56 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     val model2 = standardizer2.fit(dataRDD)
     val model3 = standardizer3.fit(dataRDD)
 
-    val equivalentModel1 = new StandardScalerModel(model1.mean, model1.variance, true, true)
-    val equivalentModel2 = new StandardScalerModel(model2.mean, model2.variance)
-    val equivalentModel3 = new StandardScalerModel(model3.mean, model3.variance, true, false)
+    val data1 = denseData.map(model1.transform)
+    val data2 = denseData.map(model2.transform)
+    val data3 = denseData.map(model3.transform)
 
-    val data1 = denseData.map(equivalentModel1.transform)
-    val data2 = denseData.map(equivalentModel2.transform)
-    val data3 = denseData.map(equivalentModel3.transform)
+    val data1RDD = model1.transform(dataRDD)
+    val data2RDD = model2.transform(dataRDD)
+    val data3RDD = model3.transform(dataRDD)
 
-    val data1RDD = equivalentModel1.transform(dataRDD)
-    val data2RDD = equivalentModel2.transform(dataRDD)
-    val data3RDD = equivalentModel3.transform(dataRDD)
+    val summary = computeSummary(dataRDD)
+    val summary1 = computeSummary(data1RDD)
+    val summary2 = computeSummary(data2RDD)
+    val summary3 = computeSummary(data3RDD)
 
-    validateDense(data1, data2, data3, dataRDD, data1RDD, data2RDD, data3RDD)
-  }
+    assert((denseData, data1, data1RDD.collect()).zipped.forall {
+      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
+      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
+      case _ => false
+    }, "The vector type should be preserved after standardization.")
 
-  test("Standardization with dense input") {
+    assert((denseData, data2, data2RDD.collect()).zipped.forall {
+      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
+      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
+      case _ => false
+    }, "The vector type should be preserved after standardization.")
 
-    val dataRDD = sc.parallelize(denseData, 3)
+    assert((denseData, data3, data3RDD.collect()).zipped.forall {
+      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
+      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
+      case _ => false
+    }, "The vector type should be preserved after standardization.")
 
-    val standardizer1 = new StandardScaler(withMean = true, withStd = true)
-    val standardizer2 = new StandardScaler()
-    val standardizer3 = new StandardScaler(withMean = true, withStd = false)
+    assert((data1, data1RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
+    assert((data2, data2RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
+    assert((data3, data3RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
 
-    val model1 = standardizer1.fit(dataRDD)
-    val model2 = standardizer2.fit(dataRDD)
-    val model3 = standardizer3.fit(dataRDD)
+    assert(summary1.mean ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
+    assert(summary1.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
 
-    val data1 = denseData.map(model1.transform)
-    val data2 = denseData.map(model2.transform)
-    val data3 = denseData.map(model3.transform)
+    assert(summary2.mean !~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
+    assert(summary2.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
 
-    val data1RDD = model1.transform(dataRDD)
-    val data2RDD = model2.transform(dataRDD)
-    val data3RDD = model3.transform(dataRDD)
+    assert(summary3.mean ~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
+    assert(summary3.variance ~== summary.variance absTol 1E-5)
 
-    validateDense(data1, data2, data3, dataRDD, data1RDD, data2RDD, data3RDD)
+    assert(data1(0) ~== Vectors.dense(-1.31527964, 1.023470449, 0.11637768424) absTol 1E-5)
+    assert(data1(3) ~== Vectors.dense(1.637735298, 0.156973995, 1.32247368462) absTol 1E-5)
+    assert(data2(4) ~== Vectors.dense(0.865538862, -0.22604255, 0.0) absTol 1E-5)
+    assert(data2(5) ~== Vectors.dense(0.0, 0.71580142, 0.0) absTol 1E-5)
+    assert(data3(1) ~== Vectors.dense(-0.58333333, -0.58333333, -2.8166666666) absTol 1E-5)
+    assert(data3(5) ~== Vectors.dense(-0.58333333, 2.316666666, 0.18333333333) absTol 1E-5)
   }
 
 
@@ -226,8 +226,21 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
 
     val data2RDD = equivalentModel2.transform(dataRDD)
 
-    validateSparse(data2, data2RDD)
+    val summary = computeSummary(data2RDD)
+
+    assert((sparseData, data2, data2RDD.collect()).zipped.forall {
+      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
+      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
+      case _ => false
+    }, "The vector type should be preserved after standardization.")
+
+    assert((data2, data2RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
+
+    assert(summary.mean !~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
+    assert(summary.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
 
+    assert(data2(4) ~== Vectors.sparse(3, Seq((0, 0.865538862), (1, -0.22604255))) absTol 1E-5)
+    assert(data2(5) ~== Vectors.sparse(3, Seq((1, 0.71580142))) absTol 1E-5)
   }
 
   test("Standardization with sparse input") {
@@ -258,7 +271,22 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
 
     val data2RDD = model2.transform(dataRDD)
 
-    validateSparse(data2, data2RDD)
+
+    val summary = computeSummary(data2RDD)
+
+    assert((sparseData, data2, data2RDD.collect()).zipped.forall {
+      case (v1: DenseVector, v2: DenseVector, v3: DenseVector) => true
+      case (v1: SparseVector, v2: SparseVector, v3: SparseVector) => true
+      case _ => false
+    }, "The vector type should be preserved after standardization.")
+
+    assert((data2, data2RDD.collect()).zipped.forall((v1, v2) => v1 ~== v2 absTol 1E-5))
+
+    assert(summary.mean !~== Vectors.dense(0.0, 0.0, 0.0) absTol 1E-5)
+    assert(summary.variance ~== Vectors.dense(1.0, 1.0, 1.0) absTol 1E-5)
+
+    assert(data2(4) ~== Vectors.sparse(3, Seq((0, 0.865538862), (1, -0.22604255))) absTol 1E-5)
+    assert(data2(5) ~== Vectors.sparse(3, Seq((1, 0.71580142))) absTol 1E-5)
   }
 
   test("Standardization with constant input when means and variances are provided") {
@@ -281,8 +309,12 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     val data2 = constantData.map(equivalentModel2.transform)
     val data3 = constantData.map(equivalentModel3.transform)
 
-    validateConstant(data1, data2, data3)
-
+    assert(data1.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
+    assert(data2.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
+    assert(data3.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
   }
 
   test("Standardization with constant input") {
@@ -301,8 +333,11 @@ class StandardScalerSuite extends FunSuite with MLlibTestSparkContext {
     val data2 = constantData.map(model2.transform)
     val data3 = constantData.map(model3.transform)
 
-    validateConstant(data1, data2, data3)
-
+    assert(data1.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
+    assert(data2.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
+    assert(data3.forall(_.toArray.forall(_ == 0.0)),
+      "The variance is zero, so the transformed result should be 0.0")
   }
-
 }