From 3820f41d6f28f6461b90b77a439363a61486c4b0 Mon Sep 17 00:00:00 2001 From: tijo Date: Mon, 31 Aug 2015 00:56:23 +0530 Subject: [PATCH 1/8] Modified for adding @Since annotation --- .../apache/spark/ml/recommendation/ALS.scala | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 7db8ad8d2791..af7200f756c6 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.netlib.util.intW import org.apache.spark.{Logging, Partitioner} -import org.apache.spark.annotation.{DeveloperApi, Experimental} +import org.apache.spark.annotation.{Since, DeveloperApi, Experimental} import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ @@ -178,6 +178,7 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w * @param itemFactors a DataFrame that stores item factors in two columns: `id` and `features` */ @Experimental +@Since("1.3.0") class ALSModel private[ml] ( override val uid: String, val rank: Int, @@ -186,14 +187,18 @@ class ALSModel private[ml] ( extends Model[ALSModel] with ALSModelParams { /** @group setParam */ + @Since("1.4.0") def setUserCol(value: String): this.type = set(userCol, value) /** @group setParam */ + @Since("1.4.0") def setItemCol(value: String): this.type = set(itemCol, value) /** @group setParam */ + @Since("1.3.0") def setPredictionCol(value: String): this.type = set(predictionCol, value) + @Since("1.3.0") override def transform(dataset: DataFrame): DataFrame = { // Register a UDF for DataFrame, and then // create a new column named map(predictionCol) by running the predict UDF. @@ -210,13 +215,13 @@ class ALSModel private[ml] ( .select(dataset("*"), predict(userFactors("features"), itemFactors("features")).as($(predictionCol))) } - + @Since("1.3.0") override def transformSchema(schema: StructType): StructType = { SchemaUtils.checkColumnType(schema, $(userCol), IntegerType) SchemaUtils.checkColumnType(schema, $(itemCol), IntegerType) SchemaUtils.appendColumn(schema, $(predictionCol), FloatType) } - + @Since("1.5.0") override def copy(extra: ParamMap): ALSModel = { val copied = new ALSModel(uid, rank, userFactors, itemFactors) copyValues(copied, extra).setParent(parent) @@ -255,64 +260,80 @@ class ALSModel private[ml] ( * preferences rather than explicit ratings given to items. */ @Experimental -class ALS(override val uid: String) extends Estimator[ALSModel] with ALSParams { +@Since("1.3.0") +class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel] with ALSParams { import org.apache.spark.ml.recommendation.ALS.Rating def this() = this(Identifiable.randomUID("als")) /** @group setParam */ + @Since("1.3.0") def setRank(value: Int): this.type = set(rank, value) /** @group setParam */ + @Since("1.3.0") def setNumUserBlocks(value: Int): this.type = set(numUserBlocks, value) /** @group setParam */ + @Since("1.3.0") def setNumItemBlocks(value: Int): this.type = set(numItemBlocks, value) /** @group setParam */ + @Since("1.3.0") def setImplicitPrefs(value: Boolean): this.type = set(implicitPrefs, value) /** @group setParam */ + @Since("1.3.0") def setAlpha(value: Double): this.type = set(alpha, value) /** @group setParam */ + @Since("1.3.0") def setUserCol(value: String): this.type = set(userCol, value) /** @group setParam */ + @Since("1.3.0") def setItemCol(value: String): this.type = set(itemCol, value) /** @group setParam */ + @Since("1.3.0") def setRatingCol(value: String): this.type = set(ratingCol, value) /** @group setParam */ + @Since("1.3.0") def setPredictionCol(value: String): this.type = set(predictionCol, value) /** @group setParam */ + @Since("1.3.0") def setMaxIter(value: Int): this.type = set(maxIter, value) /** @group setParam */ + @Since("1.3.0") def setRegParam(value: Double): this.type = set(regParam, value) /** @group setParam */ + @Since("1.3.0") def setNonnegative(value: Boolean): this.type = set(nonnegative, value) /** @group setParam */ + @Since("1.4.0") def setCheckpointInterval(value: Int): this.type = set(checkpointInterval, value) /** @group setParam */ + @Since("1.4.0") def setSeed(value: Long): this.type = set(seed, value) /** * Sets both numUserBlocks and numItemBlocks to the specific value. * @group setParam */ + @Since("1.3.0") def setNumBlocks(value: Int): this.type = { setNumUserBlocks(value) setNumItemBlocks(value) this } - + @Since("1.3.0") override def fit(dataset: DataFrame): ALSModel = { import dataset.sqlContext.implicits._ val ratings = dataset @@ -331,11 +352,11 @@ class ALS(override val uid: String) extends Estimator[ALSModel] with ALSParams { val model = new ALSModel(uid, $(rank), userDF, itemDF).setParent(this) copyValues(model) } - + @Since("1.3.0") override def transformSchema(schema: StructType): StructType = { validateAndTransformSchema(schema) } - + @Since("1.5.0") override def copy(extra: ParamMap): ALS = defaultCopy(extra) } @@ -348,6 +369,7 @@ class ALS(override val uid: String) extends Estimator[ALSModel] with ALSParams { * than 2 billion. */ @DeveloperApi +@Since("1.3.0") object ALS extends Logging { /** @@ -355,6 +377,7 @@ object ALS extends Logging { * Rating class for better code readability. */ @DeveloperApi + @Since("1.3.0") case class Rating[@specialized(Int, Long) ID](user: ID, item: ID, rating: Float) /** Trait for least squares solvers applied to the normal equation. */ @@ -426,6 +449,7 @@ object ALS extends Logging { * min_x_ norm(A x - b)^2^ + lambda * n * norm(x)^2^ * subject to x >= 0 */ + @Since("1.3.0") override def solve(ne: NormalEquation, lambda: Double): Array[Float] = { val rank = ne.k initialize(rank) @@ -519,6 +543,7 @@ object ALS extends Logging { * Implementation of the ALS algorithm. */ @DeveloperApi + @Since("1.3.0") def train[ID: ClassTag]( // scalastyle:ignore ratings: RDD[Rating[ID]], rank: Int = 10, From 57b366d52621a27cd570a6867280932893e43780 Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Mon, 31 Aug 2015 16:07:54 +0530 Subject: [PATCH 2/8] updated to add Since Annotation --- .../src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index af7200f756c6..e7aca8355a9b 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -31,7 +31,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.netlib.util.intW import org.apache.spark.{Logging, Partitioner} -import org.apache.spark.annotation.{Since, DeveloperApi, Experimental} +import org.apache.spark.annotation.{DeveloperApi, Experimental, Since} import org.apache.spark.ml.{Estimator, Model} import org.apache.spark.ml.param._ import org.apache.spark.ml.param.shared._ From 4f32939402805c492fae95279c18f270260d0d22 Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Mon, 31 Aug 2015 16:26:18 +0530 Subject: [PATCH 3/8] Modified for checkstyle fix --- .../main/scala/org/apache/spark/ml/recommendation/ALS.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index e7aca8355a9b..eb408cf68070 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -215,12 +215,14 @@ class ALSModel private[ml] ( .select(dataset("*"), predict(userFactors("features"), itemFactors("features")).as($(predictionCol))) } + @Since("1.3.0") override def transformSchema(schema: StructType): StructType = { SchemaUtils.checkColumnType(schema, $(userCol), IntegerType) SchemaUtils.checkColumnType(schema, $(itemCol), IntegerType) SchemaUtils.appendColumn(schema, $(predictionCol), FloatType) } + @Since("1.5.0") override def copy(extra: ParamMap): ALSModel = { val copied = new ALSModel(uid, rank, userFactors, itemFactors) @@ -333,6 +335,7 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel] setNumItemBlocks(value) this } + @Since("1.3.0") override def fit(dataset: DataFrame): ALSModel = { import dataset.sqlContext.implicits._ @@ -352,10 +355,12 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel] val model = new ALSModel(uid, $(rank), userDF, itemDF).setParent(this) copyValues(model) } + @Since("1.3.0") override def transformSchema(schema: StructType): StructType = { validateAndTransformSchema(schema) } + @Since("1.5.0") override def copy(extra: ParamMap): ALS = defaultCopy(extra) } From f3cc61ac5b60d0d25923bcaca9915cad8d0e2611 Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Tue, 1 Sep 2015 15:23:56 +0530 Subject: [PATCH 4/8] Added Auxiliary constructor --- .../src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index eb408cf68070..47840f62ff5b 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -267,6 +267,7 @@ class ALS(@Since("1.4.0") override val uid: String) extends Estimator[ALSModel] import org.apache.spark.ml.recommendation.ALS.Rating + @Since("1.4.0") def this() = this(Identifiable.randomUID("als")) /** @group setParam */ From 3988055cad68cf02e3721915b67a8c22baa8216e Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Tue, 1 Sep 2015 15:40:18 +0530 Subject: [PATCH 5/8] Added annotation for case class --- .../main/scala/org/apache/spark/ml/recommendation/ALS.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 47840f62ff5b..953fe65eea34 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -384,7 +384,10 @@ object ALS extends Logging { */ @DeveloperApi @Since("1.3.0") - case class Rating[@specialized(Int, Long) ID](user: ID, item: ID, rating: Float) + case class Rating @Since("1.3.0") [@specialized(Int, Long) ID]( + @Since("1.3.0") user: ID, + @Since("1.3.0") item: ID, + @Since("1.3.0") rating: Float) /** Trait for least squares solvers applied to the normal equation. */ private[recommendation] trait LeastSquaresNESolver extends Serializable { From adf64e1c7b648953b86a31aa46184e1f4f2007a8 Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Tue, 1 Sep 2015 16:04:32 +0530 Subject: [PATCH 6/8] Fixed compilation error dueto earlier fix --- .../src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 953fe65eea34..292776e881a1 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -384,7 +384,7 @@ object ALS extends Logging { */ @DeveloperApi @Since("1.3.0") - case class Rating @Since("1.3.0") [@specialized(Int, Long) ID]( + case class Rating [@specialized(Int, Long) ID] @Since("1.3.0")( @Since("1.3.0") user: ID, @Since("1.3.0") item: ID, @Since("1.3.0") rating: Float) From 624d85dd8bfc4d193f34d99eadfc29ffcc057876 Mon Sep 17 00:00:00 2001 From: Tijo Thomas Date: Tue, 1 Sep 2015 16:08:13 +0530 Subject: [PATCH 7/8] removed unwanted space --- .../src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 292776e881a1..89e573f97343 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -384,7 +384,7 @@ object ALS extends Logging { */ @DeveloperApi @Since("1.3.0") - case class Rating [@specialized(Int, Long) ID] @Since("1.3.0")( + case class Rating[@specialized(Int, Long) ID] @Since("1.3.0")( @Since("1.3.0") user: ID, @Since("1.3.0") item: ID, @Since("1.3.0") rating: Float) From 7b65e62c7ef89cf8136758fcc5b7016b28f58be2 Mon Sep 17 00:00:00 2001 From: tijo Date: Sun, 6 Sep 2015 07:07:20 +0530 Subject: [PATCH 8/8] updated review comments --- .../main/scala/org/apache/spark/ml/recommendation/ALS.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 89e573f97343..157262c24943 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -180,8 +180,8 @@ private[recommendation] trait ALSParams extends ALSModelParams with HasMaxIter w @Experimental @Since("1.3.0") class ALSModel private[ml] ( - override val uid: String, - val rank: Int, + @Since("1.4.0") override val uid: String, + @Since("1.4.0") val rank: Int, @transient val userFactors: DataFrame, @transient val itemFactors: DataFrame) extends Model[ALSModel] with ALSModelParams {