Construct scipy.sparse matrix

Author: Lewuathe

python/pyspark/mllib/tests.py

@@ -25,6 +25,7 @@
 import array as pyarray
 
 from numpy import array, array_equal, zeros, inf
+import scipy.sparse as sp
 from py4j.protocol import Py4JJavaError
 
 if sys.version_info[:2] <= (2, 6):
@@ -832,6 +833,14 @@ def test_append_bias_with_vector(self):
         self.assertEqual(ret[3], 1.0)
         self.assertEqual(type(ret), list)
 
+    def test_append_bias_with_sp_vector(self):
+        data = Vectors.sparse(3, {0: 2.0, 2: 2.0})
+        # Returned value must be scipy.sparse matrix
+        ret = MLUtils.appendBias(data)
+        self.assertEqual(ret.shape, (1, 4))
+        self.assertEqual(ret.toarray()[0][3], 1.0)
+        self.assertEqual(type(ret), sp.csc_matrix)
+
     def test_load_vectors(self):
         import shutil
         data = [

python/pyspark/mllib/util.py

@@ -17,13 +17,14 @@
 
 import sys
 import numpy as np
+import scipy.sparse as sp
 import warnings
 
 if sys.version > '3':
     xrange = range
 
 from pyspark.mllib.common import callMLlibFunc, inherit_doc
-from pyspark.mllib.linalg import Vector, Vectors, SparseVector, _convert_to_vector
+from pyspark.mllib.linalg import Vector, Vectors, DenseVector, SparseVector, _convert_to_vector
 
 
 class MLUtils(object):
@@ -176,7 +177,9 @@ def appendBias(data):
         the end of the input vector.
         """
         vec = _convert_to_vector(data)
-        if isinstance(vec, Vector):
+        if isinstance(vec, SparseVector):
+            return sp.csc_matrix(np.append(vec.toArray(), 1.0))
+        elif isinstance(vec, Vector):
             vec = vec.toArray()
         return np.append(vec, 1.0).tolist()