From 46e58da1489b0ed41784044a0738089bf7364d29 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Wed, 6 Apr 2016 14:38:10 -0700 Subject: [PATCH 1/9] add the mllib-local build to maven pom Fix MiMa Addressed some of feedback Modifying sparktestsupport/modules.py for build server fix a bug Fix the build Ignore org.apache.spark.SparkException from MiMa --- .../org/apache/spark/SparkException.scala | 0 core/pom.xml | 5 + dev/sparktestsupport/modules.py | 14 ++- mllib-local/pom.xml | 105 ++++++++++++++++++ .../ml}/linalg/CholeskyDecomposition.scala | 2 +- .../ml}/linalg/EigenValueDecomposition.scala | 4 +- .../org/apache/spark/ml/package-info.java | 22 ++++ .../scala/org/apache/spark/ml/package.scala | 50 +++++++++ .../apache/spark/ml}/util/NumericParser.scala | 4 +- .../spark/ml}/util/NumericParserSuite.scala | 2 +- mllib/pom.xml | 48 +++----- .../spark/ml/optim/WeightedLeastSquares.scala | 3 +- .../apache/spark/ml/recommendation/ALS.scala | 2 +- .../apache/spark/mllib/linalg/Vectors.scala | 2 +- .../mllib/linalg/distributed/RowMatrix.scala | 1 + .../spark/mllib/regression/LabeledPoint.scala | 2 +- pom.xml | 1 + project/MimaExcludes.scala | 1 + project/SparkBuild.scala | 6 +- 19 files changed, 225 insertions(+), 49 deletions(-) rename {core => common/sketch}/src/main/scala/org/apache/spark/SparkException.scala (100%) create mode 100644 mllib-local/pom.xml rename {mllib/src/main/scala/org/apache/spark/mllib => mllib-local/src/main/scala/org/apache/spark/ml}/linalg/CholeskyDecomposition.scala (98%) rename {mllib/src/main/scala/org/apache/spark/mllib => mllib-local/src/main/scala/org/apache/spark/ml}/linalg/EigenValueDecomposition.scala (98%) create mode 100644 mllib-local/src/main/scala/org/apache/spark/ml/package-info.java create mode 100644 mllib-local/src/main/scala/org/apache/spark/ml/package.scala rename {mllib/src/main/scala/org/apache/spark/mllib => mllib-local/src/main/scala/org/apache/spark/ml}/util/NumericParser.scala (98%) rename {mllib/src/test/scala/org/apache/spark/mllib => mllib-local/src/test/scala/org/apache/spark/ml}/util/NumericParserSuite.scala (98%) diff --git a/core/src/main/scala/org/apache/spark/SparkException.scala b/common/sketch/src/main/scala/org/apache/spark/SparkException.scala similarity index 100% rename from core/src/main/scala/org/apache/spark/SparkException.scala rename to common/sketch/src/main/scala/org/apache/spark/SparkException.scala diff --git a/core/pom.xml b/core/pom.xml index 4c7e3a36620a9..b05c639cf742c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -34,6 +34,11 @@ Spark Project Core http://spark.apache.org/ + + org.apache.spark + spark-sketch_2.11 + ${project.version} + org.apache.avro avro-mapred diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index bb04ec6ee67d2..51346709f0de0 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -256,9 +256,21 @@ def __hash__(self): ) +mllib_local = Module( + name="mllib-local", + dependencies=[sketch], + source_file_regexes=[ + "mllib-local", + ], + sbt_test_goals=[ + "mllib-local/test", + ] +) + + mllib = Module( name="mllib", - dependencies=[streaming, sql], + dependencies=[mllib_local, streaming, sql], source_file_regexes=[ "data/mllib/", "mllib/", diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml new file mode 100644 index 0000000000000..9bcdbac8e18e1 --- /dev/null +++ b/mllib-local/pom.xml @@ -0,0 +1,105 @@ + + + + + 4.0.0 + + org.apache.spark + spark-parent_2.11 + 2.0.0-SNAPSHOT + ../pom.xml + + + org.apache.spark + spark-mllib-local_2.11 + + mllib-local + + jar + Spark Project ML Local Library + http://spark.apache.org/ + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test-jar + test + + + org.apache.spark + spark-sketch_${scala.binary.version} + ${project.version} + + + org.scalanlp + breeze_${scala.binary.version} + 0.11.2 + + + + junit + junit + + + org.apache.commons + commons-math3 + + + + + org.apache.commons + commons-math3 + + + org.scalacheck + scalacheck_${scala.binary.version} + test + + + org.mockito + mockito-core + test + + + + + netlib-lgpl + + + com.github.fommil.netlib + all + ${netlib.java.version} + pom + + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala similarity index 98% rename from mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala rename to mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala index e4494792bb390..a007b160fd67f 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.mllib.linalg +package org.apache.spark.ml.linalg import com.github.fommil.netlib.LAPACK.{getInstance => lapack} import org.netlib.util.intW diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala similarity index 98% rename from mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala rename to mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala index bb94745f078e8..295788c77fe45 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.mllib.linalg +package org.apache.spark.ml.linalg import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV} import com.github.fommil.netlib.ARPACK @@ -24,7 +24,7 @@ import org.netlib.util.{doubleW, intW} /** * Compute eigen-decomposition. */ -private[mllib] object EigenValueDecomposition { +private[spark] object EigenValueDecomposition { /** * Compute the leading k eigenvalues and eigenvectors on a symmetric square matrix using ARPACK. * The caller needs to ensure that the input matrix is real symmetric. This function requires diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java b/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java new file mode 100644 index 0000000000000..870f0fe5ee793 --- /dev/null +++ b/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Spark ML is a BETA component that adds a new set of machine learning APIs to let users quickly + * assemble and configure practical machine learning pipelines. + */ +package org.apache.spark.ml; diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/package.scala b/mllib-local/src/main/scala/org/apache/spark/ml/package.scala new file mode 100644 index 0000000000000..c589d06d9f7e4 --- /dev/null +++ b/mllib-local/src/main/scala/org/apache/spark/ml/package.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +/** + * Spark ML is a BETA component that adds a new set of machine learning APIs to let users quickly + * assemble and configure practical machine learning pipelines. + * + * @groupname param Parameters + * @groupdesc param A list of (hyper-)parameter keys this algorithm can take. Users can set and get + * the parameter values through setters and getters, respectively. + * @groupprio param -5 + * + * @groupname setParam Parameter setters + * @groupprio setParam 5 + * + * @groupname getParam Parameter getters + * @groupprio getParam 6 + * + * @groupname expertParam (expert-only) Parameters + * @groupdesc expertParam A list of advanced, expert-only (hyper-)parameter keys this algorithm can + * take. Users can set and get the parameter values through setters and getters, + * respectively. + * @groupprio expertParam 7 + * + * @groupname expertSetParam (expert-only) Parameter setters + * @groupprio expertSetParam 8 + * + * @groupname expertGetParam (expert-only) Parameter getters + * @groupprio expertGetParam 9 + * + * @groupname Ungrouped Members + * @groupprio Ungrouped 0 + */ +package object ml diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala b/mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala similarity index 98% rename from mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala rename to mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala index 2c613348c2d92..e8c2bae1cc4fa 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.mllib.util +package org.apache.spark.ml.util import java.util.StringTokenizer @@ -30,7 +30,7 @@ import org.apache.spark.SparkException * - array: an array of numbers stored as `[v0,v1,...,vn]` * - tuple: a list of numbers, arrays, or tuples stored as `(...)` */ -private[mllib] object NumericParser { +private[spark] object NumericParser { /** Parses a string into a Double, an Array[Double], or a Seq[Any]. */ def parse(s: String): Any = { diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala similarity index 98% rename from mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala rename to mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala index 16d7c3ab39b03..ce5e11c4058fd 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.mllib.util +package org.apache.spark.ml.util import org.apache.spark.{SparkException, SparkFunSuite} diff --git a/mllib/pom.xml b/mllib/pom.xml index 428176dcbfadf..933d7cbfe78d2 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -63,25 +63,20 @@ ${project.version} - org.scalanlp - breeze_${scala.binary.version} - 0.11.2 - - - - junit - junit - - - org.apache.commons - commons-math3 - - + org.apache.spark + spark-mllib-local_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-mllib-local_${scala.binary.version} + ${project.version} + test-jar + test - org.apache.commons - commons-math3 + org.apache.spark + spark-test-tags_${scala.binary.version} org.scalacheck @@ -115,24 +110,7 @@ - - org.apache.spark - spark-test-tags_${scala.binary.version} - - - - - netlib-lgpl - - - com.github.fommil.netlib - all - ${netlib.java.version} - pom - - - - + target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala index 7d21302f962bf..7bf494ef36120 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala @@ -19,7 +19,8 @@ package org.apache.spark.ml.optim import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.Instance -import org.apache.spark.mllib.linalg._ +import org.apache.spark.ml.linalg.CholeskyDecomposition +import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vector} import org.apache.spark.rdd.RDD /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 4a3ad662a0d3f..5ca270d309f1e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -34,10 +34,10 @@ import org.apache.spark.Partitioner import org.apache.spark.annotation.{DeveloperApi, Experimental, Since} import org.apache.spark.internal.Logging import org.apache.spark.ml.{Estimator, Model} +import org.apache.spark.ml.linalg.CholeskyDecomposition import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ import org.apache.spark.ml.util._ -import org.apache.spark.mllib.linalg.CholeskyDecomposition import org.apache.spark.mllib.optimization.NNLS import org.apache.spark.rdd.RDD import org.apache.spark.sql.DataFrame diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index 0f0c3a2df556b..4f61fb2f2a001 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -30,7 +30,7 @@ import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} import org.apache.spark.SparkException import org.apache.spark.annotation.{AlphaComponent, Since} -import org.apache.spark.mllib.util.NumericParser +import org.apache.spark.ml.util.NumericParser import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.GenericMutableRow import org.apache.spark.sql.catalyst.util.GenericArrayData diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala index f6183a5eaadc0..b22758dfce5d9 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala @@ -27,6 +27,7 @@ import breeze.numerics.{sqrt => brzSqrt} import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging +import org.apache.spark.ml.linalg.EigenValueDecomposition import org.apache.spark.mllib.linalg._ import org.apache.spark.mllib.stat.{MultivariateOnlineSummarizer, MultivariateStatisticalSummary} import org.apache.spark.rdd.RDD diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala index 45540f0c5c4ce..bab7863a46096 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala @@ -20,8 +20,8 @@ package org.apache.spark.mllib.regression import scala.beans.BeanInfo import org.apache.spark.annotation.Since +import org.apache.spark.ml.util.NumericParser import org.apache.spark.mllib.linalg.{Vector, Vectors} -import org.apache.spark.mllib.util.NumericParser import org.apache.spark.SparkException /** diff --git a/pom.xml b/pom.xml index 984b2859efbec..081c86220c477 100644 --- a/pom.xml +++ b/pom.xml @@ -94,6 +94,7 @@ core graphx mllib + mllib-local tools streaming sql/catalyst diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index d916c49a6a4d9..f8a4270a85809 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -52,6 +52,7 @@ object MimaExcludes { // SPARK-12600 Remove SQL deprecated methods ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$QueryExecution"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$SparkPlanner"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.applySchema"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.parquetFile"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.jdbc"), diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 60124ef0a13bc..c5688ecec65d4 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -47,9 +47,9 @@ object BuildCommons { ).map(ProjectRef(buildLocation, _)) val allProjects@Seq( - core, graphx, mllib, repl, networkCommon, networkShuffle, launcher, unsafe, testTags, sketch, _* + core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, testTags, sketch, _* ) = Seq( - "core", "graphx", "mllib", "repl", "network-common", "network-shuffle", "launcher", "unsafe", + "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe", "test-tags", "sketch" ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects @@ -254,7 +254,7 @@ object SparkBuild extends PomBuild { val mimaProjects = allProjects.filterNot { x => Seq( spark, hive, hiveThriftServer, catalyst, repl, networkCommon, networkShuffle, networkYarn, - unsafe, testTags, sketch + unsafe, testTags, sketch, mllibLocal ).contains(x) } From 0e8a6090b54885a81dcf59b65ae35fc4e531176b Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 13:34:15 -0700 Subject: [PATCH 2/9] revert --- core/pom.xml | 5 -- .../org/apache/spark/SparkException.scala | 0 dev/sparktestsupport/modules.py | 2 +- mllib-local/pom.xml | 11 ---- .../org/apache/spark/ml/package-info.java | 22 -------- .../scala/org/apache/spark/ml/package.scala | 50 ------------------- .../spark/ml/optim/WeightedLeastSquares.scala | 3 +- .../apache/spark/ml/recommendation/ALS.scala | 2 +- .../apache/spark/mllib/linalg/Vectors.scala | 2 +- .../mllib/linalg/distributed/RowMatrix.scala | 1 - .../spark/mllib/regression/LabeledPoint.scala | 2 +- .../spark/mllib}/util/NumericParser.scala | 6 +-- .../mllib}/linalg/CholeskyDecomposition.scala | 4 +- .../linalg/EigenValueDecomposition.scala | 4 +- .../mllib}/util/NumericParserSuite.scala | 4 +- 15 files changed, 13 insertions(+), 105 deletions(-) rename {common/sketch => core}/src/main/scala/org/apache/spark/SparkException.scala (100%) delete mode 100644 mllib-local/src/main/scala/org/apache/spark/ml/package-info.java delete mode 100644 mllib-local/src/main/scala/org/apache/spark/ml/package.scala rename {mllib-local/src/main/scala/org/apache/spark/ml => mllib/src/main/scala/org/apache/spark/mllib}/util/NumericParser.scala (97%) rename {mllib-local/src/main/scala/org/apache/spark/ml => mllib/src/test/scala/org/apache/spark/mllib}/linalg/CholeskyDecomposition.scala (96%) rename {mllib-local/src/main/scala/org/apache/spark/ml => mllib/src/test/scala/org/apache/spark/mllib}/linalg/EigenValueDecomposition.scala (98%) rename {mllib-local/src/test/scala/org/apache/spark/ml => mllib/src/test/scala/org/apache/spark/mllib}/util/NumericParserSuite.scala (95%) diff --git a/core/pom.xml b/core/pom.xml index b05c639cf742c..4c7e3a36620a9 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -34,11 +34,6 @@ Spark Project Core http://spark.apache.org/ - - org.apache.spark - spark-sketch_2.11 - ${project.version} - org.apache.avro avro-mapred diff --git a/common/sketch/src/main/scala/org/apache/spark/SparkException.scala b/core/src/main/scala/org/apache/spark/SparkException.scala similarity index 100% rename from common/sketch/src/main/scala/org/apache/spark/SparkException.scala rename to core/src/main/scala/org/apache/spark/SparkException.scala diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py index 51346709f0de0..c844bcff7e4f0 100644 --- a/dev/sparktestsupport/modules.py +++ b/dev/sparktestsupport/modules.py @@ -258,7 +258,7 @@ def __hash__(self): mllib_local = Module( name="mllib-local", - dependencies=[sketch], + dependencies=[], source_file_regexes=[ "mllib-local", ], diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 9bcdbac8e18e1..69917eb0fb5fe 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -35,12 +35,6 @@ http://spark.apache.org/ - - org.apache.spark - spark-core_${scala.binary.version} - ${project.version} - test - org.apache.spark spark-core_${scala.binary.version} @@ -48,11 +42,6 @@ test-jar test - - org.apache.spark - spark-sketch_${scala.binary.version} - ${project.version} - org.scalanlp breeze_${scala.binary.version} diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java b/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java deleted file mode 100644 index 870f0fe5ee793..0000000000000 --- a/mllib-local/src/main/scala/org/apache/spark/ml/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Spark ML is a BETA component that adds a new set of machine learning APIs to let users quickly - * assemble and configure practical machine learning pipelines. - */ -package org.apache.spark.ml; diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/package.scala b/mllib-local/src/main/scala/org/apache/spark/ml/package.scala deleted file mode 100644 index c589d06d9f7e4..0000000000000 --- a/mllib-local/src/main/scala/org/apache/spark/ml/package.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark - -/** - * Spark ML is a BETA component that adds a new set of machine learning APIs to let users quickly - * assemble and configure practical machine learning pipelines. - * - * @groupname param Parameters - * @groupdesc param A list of (hyper-)parameter keys this algorithm can take. Users can set and get - * the parameter values through setters and getters, respectively. - * @groupprio param -5 - * - * @groupname setParam Parameter setters - * @groupprio setParam 5 - * - * @groupname getParam Parameter getters - * @groupprio getParam 6 - * - * @groupname expertParam (expert-only) Parameters - * @groupdesc expertParam A list of advanced, expert-only (hyper-)parameter keys this algorithm can - * take. Users can set and get the parameter values through setters and getters, - * respectively. - * @groupprio expertParam 7 - * - * @groupname expertSetParam (expert-only) Parameter setters - * @groupprio expertSetParam 8 - * - * @groupname expertGetParam (expert-only) Parameter getters - * @groupprio expertGetParam 9 - * - * @groupname Ungrouped Members - * @groupprio Ungrouped 0 - */ -package object ml diff --git a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala index 7bf494ef36120..7d21302f962bf 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/optim/WeightedLeastSquares.scala @@ -19,8 +19,7 @@ package org.apache.spark.ml.optim import org.apache.spark.internal.Logging import org.apache.spark.ml.feature.Instance -import org.apache.spark.ml.linalg.CholeskyDecomposition -import org.apache.spark.mllib.linalg.{BLAS, DenseVector, Vector} +import org.apache.spark.mllib.linalg._ import org.apache.spark.rdd.RDD /** diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 5ca270d309f1e..4a3ad662a0d3f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -34,10 +34,10 @@ import org.apache.spark.Partitioner import org.apache.spark.annotation.{DeveloperApi, Experimental, Since} import org.apache.spark.internal.Logging import org.apache.spark.ml.{Estimator, Model} -import org.apache.spark.ml.linalg.CholeskyDecomposition import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ import org.apache.spark.ml.util._ +import org.apache.spark.mllib.linalg.CholeskyDecomposition import org.apache.spark.mllib.optimization.NNLS import org.apache.spark.rdd.RDD import org.apache.spark.sql.DataFrame diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index 4f61fb2f2a001..0f0c3a2df556b 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -30,7 +30,7 @@ import org.json4s.jackson.JsonMethods.{compact, parse => parseJson, render} import org.apache.spark.SparkException import org.apache.spark.annotation.{AlphaComponent, Since} -import org.apache.spark.ml.util.NumericParser +import org.apache.spark.mllib.util.NumericParser import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.GenericMutableRow import org.apache.spark.sql.catalyst.util.GenericArrayData diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala index b22758dfce5d9..f6183a5eaadc0 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala @@ -27,7 +27,6 @@ import breeze.numerics.{sqrt => brzSqrt} import org.apache.spark.annotation.Since import org.apache.spark.internal.Logging -import org.apache.spark.ml.linalg.EigenValueDecomposition import org.apache.spark.mllib.linalg._ import org.apache.spark.mllib.stat.{MultivariateOnlineSummarizer, MultivariateStatisticalSummary} import org.apache.spark.rdd.RDD diff --git a/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala b/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala index bab7863a46096..45540f0c5c4ce 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/regression/LabeledPoint.scala @@ -20,8 +20,8 @@ package org.apache.spark.mllib.regression import scala.beans.BeanInfo import org.apache.spark.annotation.Since -import org.apache.spark.ml.util.NumericParser import org.apache.spark.mllib.linalg.{Vector, Vectors} +import org.apache.spark.mllib.util.NumericParser import org.apache.spark.SparkException /** diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala similarity index 97% rename from mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala rename to mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala index e8c2bae1cc4fa..e936416ed56d2 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/util/NumericParser.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.spark.ml.util +package org.apache.spark.mllib.util import java.util.StringTokenizer import scala.collection.mutable.{ArrayBuilder, ListBuffer} -import org.apache.spark.SparkException - /** * Simple parser for a numeric structure consisting of three types: * @@ -30,7 +28,7 @@ import org.apache.spark.SparkException * - array: an array of numbers stored as `[v0,v1,...,vn]` * - tuple: a list of numbers, arrays, or tuples stored as `(...)` */ -private[spark] object NumericParser { +private[mllib] object NumericParser { /** Parses a string into a Double, an Array[Double], or a Seq[Any]. */ def parse(s: String): Any = { diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala similarity index 96% rename from mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala rename to mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala index a007b160fd67f..cab7f9fa37085 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/CholeskyDecomposition.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.ml.linalg +package org.apache.spark.mllib.linalg import com.github.fommil.netlib.LAPACK.{getInstance => lapack} import org.netlib.util.intW @@ -23,7 +23,7 @@ import org.netlib.util.intW /** * Compute Cholesky decomposition. */ -private[spark] object CholeskyDecomposition { +private[mllib] object CholeskyDecomposition { /** * Solves a symmetric positive definite linear system via Cholesky factorization. diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala similarity index 98% rename from mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala rename to mllib/src/test/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala index 295788c77fe45..bb94745f078e8 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/EigenValueDecomposition.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.ml.linalg +package org.apache.spark.mllib.linalg import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV} import com.github.fommil.netlib.ARPACK @@ -24,7 +24,7 @@ import org.netlib.util.{doubleW, intW} /** * Compute eigen-decomposition. */ -private[spark] object EigenValueDecomposition { +private[mllib] object EigenValueDecomposition { /** * Compute the leading k eigenvalues and eigenvectors on a symmetric square matrix using ARPACK. * The caller needs to ensure that the input matrix is real symmetric. This function requires diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala similarity index 95% rename from mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala rename to mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala index ce5e11c4058fd..c40458a58fe57 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/util/NumericParserSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.spark.ml.util +package org.apache.spark.mllib.util -import org.apache.spark.{SparkException, SparkFunSuite} +import org.apache.spark.SparkFunSuite class NumericParserSuite extends SparkFunSuite { From aa37a642ed130085bc579d320e6b905fee4e689e Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 15:21:10 -0700 Subject: [PATCH 3/9] address feedback --- .../org/apache/spark/ml/DummyTesting.scala | 23 +++++++++++++++ .../apache/spark/ml/DummyTestingSuite.scala | 28 +++++++++++++++++++ .../mllib/linalg/CholeskyDecomposition.scala | 2 +- .../linalg/EigenValueDecomposition.scala | 0 .../spark/mllib/util/NumericParser.scala | 2 ++ .../spark/mllib/util/NumericParserSuite.scala | 2 +- 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala create mode 100644 mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala rename mllib/src/{test => main}/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala (97%) rename mllib/src/{test => main}/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala (100%) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala b/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala new file mode 100644 index 0000000000000..34cb0e17c37b1 --- /dev/null +++ b/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ml + +// This is a private class testing if the new build works. To be removed soon. +private[ml] object DummyTesting { + def add10(input: Double) = input + 10 +} diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala new file mode 100644 index 0000000000000..6c76dbfbfa9e0 --- /dev/null +++ b/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.ml + +import org.apache.spark.SparkFunSuite + +// This is testing if the new build works. To be removed soon. +class DummyTestingSuite extends SparkFunSuite { + + test("This is testing if the new build works.") { + assert(DummyTesting.add10(15) === 25) + } +} diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala similarity index 97% rename from mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala rename to mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala index cab7f9fa37085..e4494792bb390 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/CholeskyDecomposition.scala @@ -23,7 +23,7 @@ import org.netlib.util.intW /** * Compute Cholesky decomposition. */ -private[mllib] object CholeskyDecomposition { +private[spark] object CholeskyDecomposition { /** * Solves a symmetric positive definite linear system via Cholesky factorization. diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala similarity index 100% rename from mllib/src/test/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala rename to mllib/src/main/scala/org/apache/spark/mllib/linalg/EigenValueDecomposition.scala diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala index e936416ed56d2..2c613348c2d92 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/util/NumericParser.scala @@ -21,6 +21,8 @@ import java.util.StringTokenizer import scala.collection.mutable.{ArrayBuilder, ListBuffer} +import org.apache.spark.SparkException + /** * Simple parser for a numeric structure consisting of three types: * diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala index c40458a58fe57..16d7c3ab39b03 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/util/NumericParserSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.mllib.util -import org.apache.spark.SparkFunSuite +import org.apache.spark.{SparkException, SparkFunSuite} class NumericParserSuite extends SparkFunSuite { From 4b49bac976d98f82da7005c9e6b11505c0255c01 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 15:24:36 -0700 Subject: [PATCH 4/9] formating --- project/MimaExcludes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index a841246e0b2db..819459b32693e 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -52,7 +52,7 @@ object MimaExcludes { // SPARK-12600 Remove SQL deprecated methods ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$QueryExecution"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$SparkPlanner"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException"), + ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.applySchema"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.parquetFile"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.jdbc"), From 86a3ba17e8a785f77e088ce444b5149f437ef4ae Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 15:24:56 -0700 Subject: [PATCH 5/9] remove unneeded ProblemFilters.exclude[MissingClassProblem] --- project/MimaExcludes.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 819459b32693e..a53161dc9a38e 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -52,7 +52,6 @@ object MimaExcludes { // SPARK-12600 Remove SQL deprecated methods ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$QueryExecution"), ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.sql.SQLContext$SparkPlanner"), - ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.SparkException"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.applySchema"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.parquetFile"), ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.sql.SQLContext.jdbc"), From fd358f98146f628b2d9c21d70caf7dcf885ba2a2 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 15:27:24 -0700 Subject: [PATCH 6/9] scala style --- .../src/main/scala/org/apache/spark/ml/DummyTesting.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala b/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala index 34cb0e17c37b1..6b3268cdfa25c 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/DummyTesting.scala @@ -19,5 +19,5 @@ package org.apache.spark.ml // This is a private class testing if the new build works. To be removed soon. private[ml] object DummyTesting { - def add10(input: Double) = input + 10 + private[ml] def add10(input: Double): Double = input + 10 } From eba1e4d1624f0be3c5d463b92fecf2b8a8134000 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 19:58:48 -0700 Subject: [PATCH 7/9] Add back mllib dep --- mllib/pom.xml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mllib/pom.xml b/mllib/pom.xml index 933d7cbfe78d2..6ffb52650fa66 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -75,8 +75,25 @@ test - org.apache.spark - spark-test-tags_${scala.binary.version} + org.scalanlp + breeze_${scala.binary.version} + 0.11.2 + + + + junit + junit + + + org.apache.commons + commons-math3 + + + + + org.apache.commons + commons-math3 org.scalacheck From 4d6e394ae36044b97389d47d2b9dfcdbbfefde52 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Fri, 8 Apr 2016 20:06:01 -0700 Subject: [PATCH 8/9] formating --- mllib/pom.xml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mllib/pom.xml b/mllib/pom.xml index 6ffb52650fa66..e56eafc3006bd 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -127,7 +127,24 @@ - + + org.apache.spark + spark-test-tags_${scala.binary.version} + + + + + netlib-lgpl + + + com.github.fommil.netlib + all + ${netlib.java.version} + pom + + + + target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes From ec033701d0740f9633c601354123aafe7b4fb942 Mon Sep 17 00:00:00 2001 From: DB Tsai Date: Mon, 11 Apr 2016 01:37:40 -0700 Subject: [PATCH 9/9] Fix the build --- mllib-local/pom.xml | 7 ------- .../test/scala/org/apache/spark/ml/DummyTestingSuite.scala | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 69917eb0fb5fe..c56561f215926 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -35,13 +35,6 @@ http://spark.apache.org/ - - org.apache.spark - spark-core_${scala.binary.version} - ${project.version} - test-jar - test - org.scalanlp breeze_${scala.binary.version} diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala index 6c76dbfbfa9e0..51b7c2409ff22 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/DummyTestingSuite.scala @@ -17,10 +17,10 @@ package org.apache.spark.ml -import org.apache.spark.SparkFunSuite +import org.scalatest.FunSuite // scalastyle:ignore funsuite // This is testing if the new build works. To be removed soon. -class DummyTestingSuite extends SparkFunSuite { +class DummyTestingSuite extends FunSuite { // scalastyle:ignore funsuite test("This is testing if the new build works.") { assert(DummyTesting.add10(15) === 25)