Skip to content

[SPARK-10906][MLlib] More efficient SparseMatrix.equals #8960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 7 commits into from
Closed

[SPARK-10906][MLlib] More efficient SparseMatrix.equals #8960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f3f7e65
updated SparseMatrix equal method
rahulpalamuttam Oct 2, 2015
f9c6ac0
updated SparseMatrix equal method to use iterator traversal.
rahulpalamuttam Oct 9, 2015
70016e4
Updated tests to include 3x3 matrix with more zero elements.
rahulpalamuttam Oct 10, 2015
2fff7d7
Added tests with identity matrices
rahulpalamuttam Oct 10, 2015
7f3f888
unit test which fails in MatricesSuite equals
jkbradley Oct 10, 2015
b3c29ca
Updated to handle explicit and implicit zero comparison. Added corres…
rahulpalamuttam Oct 17, 2015
e882162
Merge branch 'rahulpalamuttam-RahulP_SparseMatrixEquals' of https://g…
rahulpalamuttam Oct 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,7 +550,22 @@ class SparseMatrix @Since("1.3.0") (
values: Array[Double]) = this(numRows, numCols, colPtrs, rowIndices, values, false)

override def equals(o: Any): Boolean = o match {
case m: Matrix => toBreeze == m.toBreeze
case m : Matrix =>
if (this.numRows != m.numRows || this.numCols != m.numCols) return false
if (this.numNonzeros != m.numNonzeros) return false
val activeIterator = toBreeze.activeIterator
m match {
case s: SparseMatrix =>
val filterIter = s.toBreeze.activeIterator.withFilter(_._2 != 0.0)
val currFilterIter = activeIterator.withFilter(_._2 != 0.0)
filterIter.sameElements(currFilterIter)
case d: DenseMatrix =>
while(activeIterator.hasNext){
val next = activeIterator.next()
if(d.apply(next._1._1, next._1._2) != next._2) return false
}
true
}
case _ => false
}

Expand Down
28 changes: 28 additions & 0 deletions mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ class MatricesSuite extends SparkFunSuite {
val dm2 = Matrices.dense(2, 2, Array(0.0, 2.0, 1.0, 3.0))
assert(dm1 === dm2.transpose)

val dmz = Matrices.dense(2, 2, Array(0.0, 0.0, 0.0, 0.0))
assert(dmz === dmz.transpose)

val sm1 = dm1.asInstanceOf[DenseMatrix].toSparse
assert(sm1 === sm1)
assert(sm1 === dm1)
Expand All @@ -90,6 +93,31 @@ class MatricesSuite extends SparkFunSuite {
val sm2 = dm2.asInstanceOf[DenseMatrix].toSparse
assert(sm1 === sm2.transpose)
assert(sm1 === dm2.transpose)

val smz = dmz.asInstanceOf[DenseMatrix].toSparse
val sm2z = dm2.asInstanceOf[DenseMatrix].toSparse
sm2z(0,1) = 0.0
sm2z(1,0) = 0.0
sm2z(1,1) = 0.0
assert(smz === sm2z)
assert(smz === dmz)
assert(sm2z === dmz)
assert(sm2z === smz)
val dm3 = Matrices.dense(3, 3, Array(1.0, 0.0, 4.0, 0.0, 1.0, 0.0, 3.0, 0.0, 1.0))
val sm3 = dm3.asInstanceOf[DenseMatrix].toSparse
assert(sm3 === sm3)
assert(sm3 === dm3)
assert(sm3 !== sm3.transpose)

val sm3explicit = Matrices.sparse(3, 3, colPtrs = Array(0, 3, 6, 9),
rowIndices = Array(0, 1, 2, 0, 1, 2, 0, 1, 2), values = dm3.toArray)
assert(sm3 === sm3explicit)

val dmEye = Matrices.eye(10)
val smEye = Matrices.speye(10)
assert(smEye === smEye)
assert(smEye === dmEye)
assert(smEye === smEye.transpose)
}

test("matrix copies are deep copies") {
Expand Down