moving code

Author: thunterdb

mllib/src/main/scala/org/apache/spark/ml/stat/Statistics.scala

@@ -19,7 +19,7 @@ package org.apache.spark.ml.stat
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.annotation.Since
+import org.apache.spark.annotation.{Experimental, Since}
 import org.apache.spark.ml.linalg.{SQLDataTypes, Vector}
 import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
 import org.apache.spark.mllib.stat.{Statistics => OldStatistics}
@@ -30,17 +30,18 @@ import org.apache.spark.sql.types.{StructField, StructType}
  * API for statistical functions in MLlib, compatible with Dataframes and Datasets.
  *
  * The functions in this package generalize the functions in [[org.apache.spark.sql.Dataset.stat]]
- * to MLlib's Vector types.
+ * to spark.ml's Vector types.
  */
 @Since("2.2.0")
+@Experimental
 object Statistics {
 
   /**
    * Compute the correlation matrix for the input RDD of Vectors using the specified method.
    * Methods currently supported: `pearson` (default), `spearman`.
    *
-   * @param dataset a dataset or a dataframe
-   * @param column the name of the column of vectors for which the correlation coefficient needs
+   * @param dataset A dataset or a dataframe
+   * @param column The name of the column of vectors for which the correlation coefficient needs
    *               to be computed. This must be a column of the dataset, and it must contain
    *               Vector objects.
    * @param method String specifying the method to use for computing correlation.
@@ -63,12 +64,10 @@ object Statistics {
    * which is fairly costly. Cache the input RDD before calling corr with `method = "spearman"` to
    * avoid recomputing the common lineage.
    */
-  // TODO: how do we handle missing values?
   @Since("2.2.0")
   def corr(dataset: Dataset[_], column: String, method: String): DataFrame = {
     val rdd = dataset.select(column).rdd.map {
       case Row(v: Vector) => OldVectors.fromML(v)
-//      case r: GenericRowWithSchema => OldVectors.fromML(r.getAs[Vector](0))
     }
     val oldM = OldStatistics.corr(rdd, method)
     val name = s"$method($column)"
@@ -78,8 +77,8 @@ object Statistics {
 
   /**
    * Compute the correlation matrix for the input Dataset of Vectors.
-   * @param dataset a dataset or dataframe
-   * @param column a column of this dataset
+   * @param dataset A dataset or dataframe
+   * @param column A column of this dataset
    * @return
    */
   @Since("2.2.0")

mllib/src/test/scala/org/apache/spark/ml/stat/StatisticsSuite.scala

@@ -17,19 +17,20 @@
 
 package org.apache.spark.ml.stat
 
-import breeze.linalg.{DenseMatrix => BDM, Matrix => BM}
+import breeze.linalg.{DenseMatrix => BDM}
 
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.linalg.Matrix
 import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.util.LinalgUtils
 import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.sql.{DataFrame, Row}
 
 
 class StatisticsSuite extends SparkFunSuite with MLlibTestSparkContext with Logging {
 
-  import StatisticsSuite._
+  import LinalgUtils._
 
   val xData = Array(1.0, 0.0, -2.0)
   val yData = Array(4.0, 5.0, 3.0)
@@ -77,26 +78,3 @@ class StatisticsSuite extends SparkFunSuite with MLlibTestSparkContext with Logg
   }
 
 }
-
-
-object StatisticsSuite extends Logging {
-
-  def approxEqual(v1: Double, v2: Double, threshold: Double = 1e-6): Boolean = {
-    if (v1.isNaN) {
-      v2.isNaN
-    } else {
-      math.abs(v1 - v2) <= threshold
-    }
-  }
-
-  def matrixApproxEqual(A: BM[Double], B: BM[Double], threshold: Double = 1e-6): Boolean = {
-    for (i <- 0 until A.rows; j <- 0 until A.cols) {
-      if (!approxEqual(A(i, j), B(i, j), threshold)) {
-        logInfo("i, j = " + i + ", " + j + " actual: " + A(i, j) + " expected:" + B(i, j))
-        return false
-      }
-    }
-    true
-  }
-
-}

mllib/src/test/scala/org/apache/spark/ml/util/LinalgUtils.scala

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.util
+
+import breeze.linalg.{Matrix => BM}
+
+import org.apache.spark.internal.Logging
+
+/**
+ * Utility test methods for linear algebra.
+ */
+object LinalgUtils extends Logging {
+
+
+  /**
+   * Returns true if two numbers are equal up to some tolerance.
+   */
+  def approxEqual(v1: Double, v2: Double, threshold: Double = 1e-6): Boolean = {
+    if (v1.isNaN) {
+      v2.isNaN
+    } else {
+      math.abs(v1 - v2) <= threshold
+    }
+  }
+
+  /**
+   * Returns true if two numbers are equal coefficient-wise up to some tolerance.
+   */
+  def matrixApproxEqual(A: BM[Double], B: BM[Double], threshold: Double = 1e-6): Boolean = {
+    for (i <- 0 until A.rows; j <- 0 until A.cols) {
+      if (!approxEqual(A(i, j), B(i, j), threshold)) {
+        logInfo("i, j = " + i + ", " + j + " actual: " + A(i, j) + " expected:" + B(i, j))
+        return false
+      }
+    }
+    true
+  }
+
+}