From f906c3c4abbd0e4542b66c27a394de47afdf1d75 Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Tue, 17 Sep 2019 11:00:44 -0400 Subject: [PATCH 1/6] support dot product in ml.linalg.Vectors --- .../org/apache/spark/ml/linalg/Vectors.scala | 14 +++++++++++ .../apache/spark/ml/linalg/VectorsSuite.scala | 25 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala index 6e43d60bd03a3..4b1869c90f10d 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala @@ -178,6 +178,13 @@ sealed trait Vector extends Serializable { */ @Since("2.0.0") def argmax: Int + + /** + * Calculate the dot product of this vector with another. + * + * If `size` does not match an [IllegalArgumentException] is thrown. + */ + def dot(v: Vector): Double = BLAS.dot(this, v) } /** @@ -239,6 +246,13 @@ object Vectors { }.toSeq) } + /** + * Calculate the dot product of two vectors. + * + * If `size` does not match an [IllegalArgumentException] is thrown. + */ + def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) + /** * Creates a vector of all zeros. * diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala index 0a316f57f811b..d2ba12ff69bc9 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala @@ -380,4 +380,29 @@ class VectorsSuite extends SparkMLFunSuite { Vectors.sparse(-1, Array((1, 2.0))) } } + + test("dot product only supports vectors of same size") { + intercept[IllegalArgumentException]{ Vectors.dot(Vectors.dense(arr), Vectors.zeros(1)) } + } + + test("dense vector dot product") { + val dv = Vectors.dense(arr) + assert(Vectors.dot(dv, dv) === 0.26) + assert(dv.dot(dv) === 0.26) + } + + test("sparse vector dot product") { + val sv = Vectors.sparse(n, indices, values) + assert(Vectors.dot(sv, sv) === 0.26) + assert(sv.dot(sv) === 0.26) + } + + test("mixed sparse and dense vector dot product") { + val sv = Vectors.sparse(n, indices, values) + val dv = Vectors.dense(arr) + assert(Vectors.dot(sv, dv) === 0.26) + assert(Vectors.dot(dv, sv) === 0.26) + assert(sv.dot(dv) === 0.26) + assert(dv.dot(sv) === 0.26) + } } From 2f736843f348f85e123bdf553a58f5f3f9bdb341 Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Tue, 17 Sep 2019 11:14:13 -0400 Subject: [PATCH 2/6] support dot product in mllib.linalg.Vectors --- .../apache/spark/mllib/linalg/Vectors.scala | 14 +++++++++++ .../spark/mllib/linalg/VectorsSuite.scala | 25 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index b754fad0c1796..da551313f9476 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -204,6 +204,13 @@ sealed trait Vector extends Serializable { */ @Since("2.0.0") def asML: newlinalg.Vector + + /** + * Calculate the dot product of this vector with another. + * + * If `size` does not match an [IllegalArgumentException] is thrown. + */ + def dot(v: Vector): Double = BLAS.dot(this, v) } /** @@ -351,6 +358,13 @@ object Vectors { }.toSeq) } + /** + * Calculate the dot product of two vectors. + * + * If `size` does not match an [IllegalArgumentException] is thrown. + */ + def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) + /** * Creates a vector of all zeros. * diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala index fee0b02bf8ed8..6240b7318f66a 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala @@ -510,4 +510,29 @@ class VectorsSuite extends SparkFunSuite with Logging { Vectors.sparse(-1, Array((1, 2.0))) } } + + test("dot product only supports vectors of same size") { + intercept[IllegalArgumentException]{ Vectors.dot(Vectors.dense(arr), Vectors.zeros(1)) } + } + + test("dense vector dot product") { + val dv = Vectors.dense(arr) + assert(Vectors.dot(dv, dv) === 0.26) + assert(dv.dot(dv) === 0.26) + } + + test("sparse vector dot product") { + val sv = Vectors.sparse(n, indices, values) + assert(Vectors.dot(sv, sv) === 0.26) + assert(sv.dot(sv) === 0.26) + } + + test("mixed sparse and dense vector dot product") { + val sv = Vectors.sparse(n, indices, values) + val dv = Vectors.dense(arr) + assert(Vectors.dot(sv, dv) === 0.26) + assert(Vectors.dot(dv, sv) === 0.26) + assert(sv.dot(dv) === 0.26) + assert(dv.dot(sv) === 0.26) + } } From db41e0b7f3e166bbd98a71945374c8c931706a12 Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Wed, 18 Sep 2019 10:58:02 -0400 Subject: [PATCH 3/6] add annotations and use double brackets for types in doc --- .../src/main/scala/org/apache/spark/ml/linalg/Vectors.scala | 6 ++++-- .../main/scala/org/apache/spark/mllib/linalg/Vectors.scala | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala index 4b1869c90f10d..a73259ab1b339 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala @@ -182,8 +182,9 @@ sealed trait Vector extends Serializable { /** * Calculate the dot product of this vector with another. * - * If `size` does not match an [IllegalArgumentException] is thrown. + * If `size` does not match an [[IllegalArgumentException]] is thrown. */ + @Since("3.0.0") def dot(v: Vector): Double = BLAS.dot(this, v) } @@ -249,8 +250,9 @@ object Vectors { /** * Calculate the dot product of two vectors. * - * If `size` does not match an [IllegalArgumentException] is thrown. + * If `size` does not match an [[IllegalArgumentException]] is thrown. */ + @Since("3.0.0") def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) /** diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index da551313f9476..832afdce08c8c 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -208,8 +208,9 @@ sealed trait Vector extends Serializable { /** * Calculate the dot product of this vector with another. * - * If `size` does not match an [IllegalArgumentException] is thrown. + * If `size` does not match an [[IllegalArgumentException]] is thrown. */ + @Since("3.0.0") def dot(v: Vector): Double = BLAS.dot(this, v) } @@ -361,8 +362,9 @@ object Vectors { /** * Calculate the dot product of two vectors. * - * If `size` does not match an [IllegalArgumentException] is thrown. + * If `size` does not match an [[IllegalArgumentException]] is thrown. */ + @Since("3.0.0") def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) /** From 475922dfd0f9da03a33dd262b32ea5a471ee1803 Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Wed, 18 Sep 2019 11:35:31 -0400 Subject: [PATCH 4/6] use Vector dot method for Vectors.dot --- .../src/main/scala/org/apache/spark/ml/linalg/Vectors.scala | 2 +- .../src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala index a73259ab1b339..aa3db7f8d8003 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala @@ -253,7 +253,7 @@ object Vectors { * If `size` does not match an [[IllegalArgumentException]] is thrown. */ @Since("3.0.0") - def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) + def dot(v1: Vector, v2: Vector): Double = v1.dot(v2) /** * Creates a vector of all zeros. diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index 832afdce08c8c..7634c0ad840f9 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -365,7 +365,7 @@ object Vectors { * If `size` does not match an [[IllegalArgumentException]] is thrown. */ @Since("3.0.0") - def dot(v1: Vector, v2: Vector): Double = BLAS.dot(v1, v2) + def dot(v1: Vector, v2: Vector): Double = v1.dot(v2) /** * Creates a vector of all zeros. From 7cd1fc55e12f3d0f1ce867fc0eaa9cd11f5f612c Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Wed, 18 Sep 2019 11:36:33 -0400 Subject: [PATCH 5/6] test Vector dot method for IllegalArgumentException --- .../test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala | 5 ++++- .../scala/org/apache/spark/mllib/linalg/VectorsSuite.scala | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala index d2ba12ff69bc9..9ff777120330e 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala @@ -382,7 +382,10 @@ class VectorsSuite extends SparkMLFunSuite { } test("dot product only supports vectors of same size") { - intercept[IllegalArgumentException]{ Vectors.dot(Vectors.dense(arr), Vectors.zeros(1)) } + val vSize4 = Vectors.dense(arr) + val vSize1 = Vectors.zeros(1) + intercept[IllegalArgumentException]{ Vectors.dot(vSize4, vSize1) } + intercept[IllegalArgumentException]{ vSize1.dot(vSize4) } } test("dense vector dot product") { diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala index 6240b7318f66a..87a02a11c1a9f 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala @@ -512,7 +512,10 @@ class VectorsSuite extends SparkFunSuite with Logging { } test("dot product only supports vectors of same size") { - intercept[IllegalArgumentException]{ Vectors.dot(Vectors.dense(arr), Vectors.zeros(1)) } + val vSize4 = Vectors.dense(arr) + val vSize1 = Vectors.zeros(1) + intercept[IllegalArgumentException]{ Vectors.dot(vSize4, vSize1) } + intercept[IllegalArgumentException]{ vSize1.dot(vSize4) } } test("dense vector dot product") { From 9d2959b53b79deedd58ec6c51a978bb95daabc25 Mon Sep 17 00:00:00 2001 From: Patrick Pisciuneri Date: Thu, 19 Sep 2019 12:37:15 -0400 Subject: [PATCH 6/6] delete extra routines --- .../main/scala/org/apache/spark/ml/linalg/Vectors.scala | 8 -------- .../scala/org/apache/spark/ml/linalg/VectorsSuite.scala | 5 ----- .../scala/org/apache/spark/mllib/linalg/Vectors.scala | 8 -------- .../org/apache/spark/mllib/linalg/VectorsSuite.scala | 5 ----- 4 files changed, 26 deletions(-) diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala index aa3db7f8d8003..f437d66cddb54 100644 --- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala +++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala @@ -247,14 +247,6 @@ object Vectors { }.toSeq) } - /** - * Calculate the dot product of two vectors. - * - * If `size` does not match an [[IllegalArgumentException]] is thrown. - */ - @Since("3.0.0") - def dot(v1: Vector, v2: Vector): Double = v1.dot(v2) - /** * Creates a vector of all zeros. * diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala index 9ff777120330e..c97dc2c3c06f8 100644 --- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala +++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala @@ -384,27 +384,22 @@ class VectorsSuite extends SparkMLFunSuite { test("dot product only supports vectors of same size") { val vSize4 = Vectors.dense(arr) val vSize1 = Vectors.zeros(1) - intercept[IllegalArgumentException]{ Vectors.dot(vSize4, vSize1) } intercept[IllegalArgumentException]{ vSize1.dot(vSize4) } } test("dense vector dot product") { val dv = Vectors.dense(arr) - assert(Vectors.dot(dv, dv) === 0.26) assert(dv.dot(dv) === 0.26) } test("sparse vector dot product") { val sv = Vectors.sparse(n, indices, values) - assert(Vectors.dot(sv, sv) === 0.26) assert(sv.dot(sv) === 0.26) } test("mixed sparse and dense vector dot product") { val sv = Vectors.sparse(n, indices, values) val dv = Vectors.dense(arr) - assert(Vectors.dot(sv, dv) === 0.26) - assert(Vectors.dot(dv, sv) === 0.26) assert(sv.dot(dv) === 0.26) assert(dv.dot(sv) === 0.26) } diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala index 7634c0ad840f9..83a519326df75 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala @@ -359,14 +359,6 @@ object Vectors { }.toSeq) } - /** - * Calculate the dot product of two vectors. - * - * If `size` does not match an [[IllegalArgumentException]] is thrown. - */ - @Since("3.0.0") - def dot(v1: Vector, v2: Vector): Double = v1.dot(v2) - /** * Creates a vector of all zeros. * diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala index 87a02a11c1a9f..b2163b518dbd1 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala @@ -514,27 +514,22 @@ class VectorsSuite extends SparkFunSuite with Logging { test("dot product only supports vectors of same size") { val vSize4 = Vectors.dense(arr) val vSize1 = Vectors.zeros(1) - intercept[IllegalArgumentException]{ Vectors.dot(vSize4, vSize1) } intercept[IllegalArgumentException]{ vSize1.dot(vSize4) } } test("dense vector dot product") { val dv = Vectors.dense(arr) - assert(Vectors.dot(dv, dv) === 0.26) assert(dv.dot(dv) === 0.26) } test("sparse vector dot product") { val sv = Vectors.sparse(n, indices, values) - assert(Vectors.dot(sv, sv) === 0.26) assert(sv.dot(sv) === 0.26) } test("mixed sparse and dense vector dot product") { val sv = Vectors.sparse(n, indices, values) val dv = Vectors.dense(arr) - assert(Vectors.dot(sv, dv) === 0.26) - assert(Vectors.dot(dv, sv) === 0.26) assert(sv.dot(dv) === 0.26) assert(dv.dot(sv) === 0.26) }