From 59a1ad1a0d9a898846a385e5ceeeb36cd87b9d57 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Tue, 21 Jun 2016 15:36:41 +0200 Subject: [PATCH 01/15] update annotation for 'clusterCenters' --- .../src/main/scala/org/apache/spark/ml/clustering/KMeans.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala index 790ef1fe8dc94..8b5cd90c05b75 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala @@ -131,7 +131,7 @@ class KMeansModel private[ml] ( private[clustering] def predict(features: Vector): Int = parentModel.predict(features) - @Since("1.5.0") + @Since("2.0.0") def clusterCenters: Array[Vector] = parentModel.clusterCenters.map(_.asML) /** From d30a955be55bff515de45d9a2726f7de67ff3bce Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Tue, 21 Jun 2016 15:36:58 +0200 Subject: [PATCH 02/15] update annotations for 'topicsMatrix' --- mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala index 609e50eb494ce..32a09626161c4 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala @@ -444,7 +444,7 @@ sealed abstract class LDAModel private[ml] ( * the Expectation-Maximization ("em") [[optimizer]], then this method could involve * collecting a large amount of data to the driver (on the order of vocabSize x k). */ - @Since("1.6.0") + @Since("2.0.0") def topicsMatrix: Matrix = oldLocalModel.topicsMatrix.asML /** Indicates whether this instance is of type [[DistributedLDAModel]] */ From 3743dbc3de2df0ce0deae612c16c43ff229b8cd3 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:15:00 +0200 Subject: [PATCH 03/15] LR coefficient --- .../org/apache/spark/ml/classification/LogisticRegression.scala | 2 +- python/pyspark/ml/classification.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala index 9469acf62e13d..3f00a9e4fa347 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala @@ -483,7 +483,7 @@ object LogisticRegression extends DefaultParamsReadable[LogisticRegression] { @Experimental class LogisticRegressionModel private[spark] ( @Since("1.4.0") override val uid: String, - @Since("1.6.0") val coefficients: Vector, + @Since("2.0.0") val coefficients: Vector, @Since("1.3.0") val intercept: Double) extends ProbabilisticClassificationModel[Vector, LogisticRegressionModel] with LogisticRegressionParams with MLWritable { diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index a3cd91790c42e..a317c5f098190 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -214,7 +214,7 @@ class LogisticRegressionModel(JavaModel, JavaMLWritable, JavaMLReadable): """ @property - @since("1.6.0") + @since("2.0.0") def coefficients(self): """ Model coefficients. From ef901fbefd7851610ce2fbd462db903bc6c4aa08 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:15:36 +0200 Subject: [PATCH 04/15] MLPC model weights --- .../ml/classification/MultilayerPerceptronClassifier.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala index c4e882240ffd2..f93e0551bb68a 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/MultilayerPerceptronClassifier.scala @@ -296,7 +296,7 @@ object MultilayerPerceptronClassifier class MultilayerPerceptronClassificationModel private[ml] ( @Since("1.5.0") override val uid: String, @Since("1.5.0") val layers: Array[Int], - @Since("1.5.0") val weights: Vector) + @Since("2.0.0") val weights: Vector) extends PredictionModel[Vector, MultilayerPerceptronClassificationModel] with Serializable with MLWritable { From 62e97eb8abf54127a58d28bfade67a181c859b0b Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:15:54 +0200 Subject: [PATCH 05/15] LDAModel doc concentration --- mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala index 32a09626161c4..b333d59258238 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala @@ -432,7 +432,7 @@ sealed abstract class LDAModel private[ml] ( * If Online LDA was used and [[optimizeDocConcentration]] was set to false, * then this returns the fixed (given) value for the [[docConcentration]] parameter. */ - @Since("1.6.0") + @Since("2.0.0") def estimatedDocConcentration: Vector = getModel.docConcentration /** From fef5d99b37711650eab918a51cef024224083d0e Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:17:07 +0200 Subject: [PATCH 06/15] NB model parameters --- .../scala/org/apache/spark/ml/classification/NaiveBayes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala index a98bdeca6b723..d034940499e8d 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/classification/NaiveBayes.scala @@ -130,8 +130,8 @@ object NaiveBayes extends DefaultParamsReadable[NaiveBayes] { @Experimental class NaiveBayesModel private[ml] ( @Since("1.5.0") override val uid: String, - @Since("1.5.0") val pi: Vector, - @Since("1.5.0") val theta: Matrix) + @Since("2.0.0") val pi: Vector, + @Since("2.0.0") val theta: Matrix) extends ProbabilisticClassificationModel[Vector, NaiveBayesModel] with NaiveBayesParams with MLWritable { From 0dec4c3274bdb7c192a1e3f007105d3312f52c8d Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:19:17 +0200 Subject: [PATCH 07/15] revert annotation update for ElementwiseProduct concrete class --- .../org/apache/spark/ml/feature/ElementwiseProduct.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/ElementwiseProduct.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/ElementwiseProduct.scala index 92fefb1e6c0f3..d07833e5805df 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/ElementwiseProduct.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/ElementwiseProduct.scala @@ -33,11 +33,11 @@ import org.apache.spark.sql.types.DataType * multiplier. */ @Experimental -@Since("2.0.0") -class ElementwiseProduct @Since("2.0.0") (@Since("2.0.0") override val uid: String) +@Since("1.4.0") +class ElementwiseProduct @Since("1.4.0") (@Since("1.4.0") override val uid: String) extends UnaryTransformer[Vector, Vector, ElementwiseProduct] with DefaultParamsWritable { - @Since("2.0.0") + @Since("1.4.0") def this() = this(Identifiable.randomUID("elemProd")) /** From 2b46931febd038f6c3b823df72bf5fb8af2c933f Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:22:34 +0200 Subject: [PATCH 08/15] revert annotation update for Normalizer concrete class --- .../org/apache/spark/ml/feature/Normalizer.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala index 9a4e682890f12..f9cbad90c9f3f 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala @@ -31,11 +31,11 @@ import org.apache.spark.sql.types.DataType * Normalize a vector to have unit norm using the given p-norm. */ @Experimental -@Since("2.0.0") -class Normalizer @Since("2.0.0") (@Since("2.0.0") override val uid: String) +@Since("1.4.0") +class Normalizer @Since("1.4.0") (@Since("1.4.0") override val uid: String) extends UnaryTransformer[Vector, Vector, Normalizer] with DefaultParamsWritable { - @Since("2.0.0") + @Since("1.4.0") def this() = this(Identifiable.randomUID("normalizer")) /** @@ -43,17 +43,17 @@ class Normalizer @Since("2.0.0") (@Since("2.0.0") override val uid: String) * (default: p = 2) * @group param */ - @Since("2.0.0") + @Since("1.4.0") val p = new DoubleParam(this, "p", "the p norm value", ParamValidators.gtEq(1)) setDefault(p -> 2.0) /** @group getParam */ - @Since("2.0.0") + @Since("1.4.0") def getP: Double = $(p) /** @group setParam */ - @Since("2.0.0") + @Since("1.4.0") def setP(value: Double): this.type = set(p, value) override protected def createTransformFunc: Vector => Vector = { From 46531167700012f50aec440dacdc1073b8a5e30c Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:23:59 +0200 Subject: [PATCH 09/15] revert annotation update for PolynomialExpansion concrete class --- .../spark/ml/feature/PolynomialExpansion.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala index 026014c7d64e3..7b35fdeaf40c6 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/PolynomialExpansion.scala @@ -35,11 +35,11 @@ import org.apache.spark.sql.types.DataType * `(x, y)`, if we want to expand it with degree 2, then we get `(x, x * x, y, x * y, y * y)`. */ @Experimental -@Since("2.0.0") -class PolynomialExpansion @Since("2.0.0") (@Since("2.0.0") override val uid: String) +@Since("1.4.0") +class PolynomialExpansion @Since("1.4.0") (@Since("1.4.0") override val uid: String) extends UnaryTransformer[Vector, Vector, PolynomialExpansion] with DefaultParamsWritable { - @Since("2.0.0") + @Since("1.4.0") def this() = this(Identifiable.randomUID("poly")) /** @@ -47,18 +47,18 @@ class PolynomialExpansion @Since("2.0.0") (@Since("2.0.0") override val uid: Str * Default: 2 * @group param */ - @Since("2.0.0") + @Since("1.4.0") val degree = new IntParam(this, "degree", "the polynomial degree to expand (>= 1)", ParamValidators.gtEq(1)) setDefault(degree -> 2) /** @group getParam */ - @Since("2.0.0") + @Since("1.4.0") def getDegree: Int = $(degree) /** @group setParam */ - @Since("2.0.0") + @Since("1.4.0") def setDegree(value: Int): this.type = set(degree, value) override protected def createTransformFunc: Vector => Vector = { v => From f5286ee0dd86fc6cdacf70387afd651ef19c04a9 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:34:07 +0200 Subject: [PATCH 10/15] Word2VecModel findSynonyms --- mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala index 05c4f2f1a7891..a4119f5f3f2ac 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Word2Vec.scala @@ -240,7 +240,7 @@ class Word2VecModel private[ml] ( * of the word. Returns a dataframe with the words and the cosine similarities between the * synonyms and the given word vector. */ - @Since("1.5.0") + @Since("2.0.0") def findSynonyms(word: Vector, num: Int): DataFrame = { val spark = SparkSession.builder().getOrCreate() spark.createDataFrame(wordVectors.findSynonyms(word, num)).toDF("word", "similarity") From 5a2114ac94cf3cc52155084299ca7d37db065ee8 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:36:24 +0200 Subject: [PATCH 11/15] aft regression model update annotations --- .../apache/spark/ml/regression/AFTSurvivalRegression.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala index 7f57af19e9df9..bf929f0c6b823 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala @@ -286,7 +286,7 @@ object AFTSurvivalRegression extends DefaultParamsReadable[AFTSurvivalRegression @Since("1.6.0") class AFTSurvivalRegressionModel private[ml] ( @Since("1.6.0") override val uid: String, - @Since("1.6.0") val coefficients: Vector, + @Since("2.0.0") val coefficients: Vector, @Since("1.6.0") val intercept: Double, @Since("1.6.0") val scale: Double) extends Model[AFTSurvivalRegressionModel] with AFTSurvivalRegressionParams with MLWritable { @@ -307,7 +307,7 @@ class AFTSurvivalRegressionModel private[ml] ( @Since("1.6.0") def setQuantilesCol(value: String): this.type = set(quantilesCol, value) - @Since("1.6.0") + @Since("2.0.0") def predictQuantiles(features: Vector): Vector = { // scale parameter for the Weibull distribution of lifetime val lambda = math.exp(BLAS.dot(coefficients, features) + intercept) @@ -319,7 +319,7 @@ class AFTSurvivalRegressionModel private[ml] ( Vectors.dense(quantiles) } - @Since("1.6.0") + @Since("2.0.0") def predict(features: Vector): Double = { math.exp(BLAS.dot(coefficients, features) + intercept) } From 1c6e2952e6680e348967ab4322f72830f44304a5 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:36:36 +0200 Subject: [PATCH 12/15] isotonic regression model update annotations --- .../org/apache/spark/ml/regression/IsotonicRegression.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala index d16e8e3f6b259..64e63695662ea 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala @@ -221,14 +221,14 @@ class IsotonicRegressionModel private[ml] ( def setFeatureIndex(value: Int): this.type = set(featureIndex, value) /** Boundaries in increasing order for which predictions are known. */ - @Since("1.5.0") + @Since("2.0.0") def boundaries: Vector = Vectors.dense(oldModel.boundaries) /** * Predictions associated with the boundaries at the same index, monotone because of isotonic * regression. */ - @Since("1.5.0") + @Since("2.0.0") def predictions: Vector = Vectors.dense(oldModel.predictions) @Since("1.5.0") From c1fab46159503767d0d9b373b367807e9aa63208 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:43:29 +0200 Subject: [PATCH 13/15] linear regression model update annotations --- .../org/apache/spark/ml/regression/LinearRegression.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala index 52ec40e15b213..c29236a36d626 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/regression/LinearRegression.scala @@ -387,9 +387,9 @@ object LinearRegression extends DefaultParamsReadable[LinearRegression] { @Since("1.3.0") @Experimental class LinearRegressionModel private[ml] ( - override val uid: String, - val coefficients: Vector, - val intercept: Double) + @Since("1.4.0") override val uid: String, + @Since("2.0.0") val coefficients: Vector, + @Since("1.3.0") val intercept: Double) extends RegressionModel[Vector, LinearRegressionModel] with LinearRegressionParams with MLWritable { From f9e4e860d5963b73216324f969ec5c96a261fc90 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:44:34 +0200 Subject: [PATCH 14/15] py linear regression model update annotations --- python/pyspark/ml/regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index 8d2378d51fb7e..50441155e8fa7 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -138,7 +138,7 @@ class LinearRegressionModel(JavaModel, JavaMLWritable, JavaMLReadable): """ @property - @since("1.6.0") + @since("2.0.0") def coefficients(self): """ Model coefficients. From 2c819a0ce0c70e02e34681cb68a038a52b2f1468 Mon Sep 17 00:00:00 2001 From: Nick Pentreath Date: Wed, 22 Jun 2016 10:52:58 +0200 Subject: [PATCH 15/15] update py regression and classification models --- python/pyspark/ml/classification.py | 6 +++--- python/pyspark/ml/regression.py | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py index a317c5f098190..79e9acfc71215 100644 --- a/python/pyspark/ml/classification.py +++ b/python/pyspark/ml/classification.py @@ -1025,7 +1025,7 @@ class NaiveBayesModel(JavaModel, JavaMLWritable, JavaMLReadable): """ @property - @since("1.5.0") + @since("2.0.0") def pi(self): """ log of class priors. @@ -1033,7 +1033,7 @@ def pi(self): return self._call_java("pi") @property - @since("1.5.0") + @since("2.0.0") def theta(self): """ log of class conditional probabilities. @@ -1230,7 +1230,7 @@ def layers(self): return self._call_java("javaLayers") @property - @since("1.6.0") + @since("2.0.0") def weights(self): """ vector of initial weights for the model that consists of the weights of layers. diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py index 50441155e8fa7..32e7605f91875 100644 --- a/python/pyspark/ml/regression.py +++ b/python/pyspark/ml/regression.py @@ -511,6 +511,7 @@ class IsotonicRegressionModel(JavaModel, JavaMLWritable, JavaMLReadable): """ @property + @since("2.0.0") def boundaries(self): """ Boundaries in increasing order for which predictions are known. @@ -518,6 +519,7 @@ def boundaries(self): return self._call_java("boundaries") @property + @since("2.0.0") def predictions(self): """ Predictions associated with the boundaries at the same index, monotone because of isotonic @@ -1248,7 +1250,7 @@ class AFTSurvivalRegressionModel(JavaModel, JavaMLWritable, JavaMLReadable): """ @property - @since("1.6.0") + @since("2.0.0") def coefficients(self): """ Model coefficients. @@ -1271,12 +1273,14 @@ def scale(self): """ return self._call_java("scale") + @since("2.0.0") def predictQuantiles(self, features): """ Predicted Quantiles """ return self._call_java("predictQuantiles", features) + @since("2.0.0") def predict(self, features): """ Predicted value