Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
f3fa658
work
thunterdb Mar 3, 2017
7539835
work on the test suite
thunterdb Mar 6, 2017
673943f
last work
thunterdb Mar 7, 2017
202b672
work on using imperative aggregators
thunterdb Mar 13, 2017
be01981
Merge remote-tracking branch 'upstream/master' into 19634
thunterdb Mar 17, 2017
a983284
more work on summarizer
thunterdb Mar 18, 2017
647a4fe
work
thunterdb Mar 21, 2017
3c4bef7
changes
thunterdb Mar 21, 2017
56390cc
Merge remote-tracking branch 'upstream/master' into 19634
thunterdb Mar 21, 2017
c3f236c
cleanup
thunterdb Mar 21, 2017
ef955c0
debugging
thunterdb Mar 21, 2017
a04f923
work
thunterdb Mar 21, 2017
946d490
Merge remote-tracking branch 'upstream/master' into 19634
thunterdb Mar 22, 2017
201eb77
debug
thunterdb Mar 22, 2017
f4dec88
trying to debug serialization issue
thunterdb Mar 23, 2017
4af0f47
better tests
thunterdb Mar 23, 2017
9f29030
changes
thunterdb Mar 24, 2017
e9877dc
debugging
thunterdb Mar 24, 2017
3a11d02
more tests and debugging
thunterdb Mar 24, 2017
6d26c17
fixed tests
thunterdb Mar 24, 2017
35eaeb0
doc
thunterdb Mar 24, 2017
58b17dc
cleanups
thunterdb Mar 24, 2017
18078c1
cleanups
thunterdb Mar 24, 2017
ffe5cfe
Cleanups
thunterdb Mar 24, 2017
41f4be6
Cleanups
thunterdb Mar 24, 2017
ba200bb
Cleanups
thunterdb Mar 24, 2017
2f809ef
Merge remote-tracking branch 'upstream/master' into 19634
thunterdb Mar 27, 2017
662f62c
small test to find perf issues
thunterdb Mar 28, 2017
96be071
Current speed:
thunterdb Mar 30, 2017
a569dac
BLAS calls for dense interface
thunterdb Mar 30, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,7 @@ import org.apache.spark.sql.types._
*/
private[spark] class VectorUDT extends UserDefinedType[Vector] {

override def sqlType: StructType = {
// type: 0 = sparse, 1 = dense
// We only use "values" for dense vectors, and "size", "indices", and "values" for sparse
// vectors. The "values" field is nullable because we might want to add binary vectors later,
// which uses "size" and "indices", but not "values".
StructType(Seq(
StructField("type", ByteType, nullable = false),
StructField("size", IntegerType, nullable = true),
StructField("indices", ArrayType(IntegerType, containsNull = false), nullable = true),
StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true)))
}
override final def sqlType: StructType = _sqlType

override def serialize(obj: Vector): InternalRow = {
obj match {
Expand Down Expand Up @@ -94,4 +84,16 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
override def typeName: String = "vector"

private[spark] override def asNullable: VectorUDT = this

private[this] val _sqlType = {
// type: 0 = sparse, 1 = dense
// We only use "values" for dense vectors, and "size", "indices", and "values" for sparse
// vectors. The "values" field is nullable because we might want to add binary vectors later,
// which uses "size" and "indices", but not "values".
StructType(Seq(
StructField("type", ByteType, nullable = false),
StructField("size", IntegerType, nullable = true),
StructField("indices", ArrayType(IntegerType, containsNull = false), nullable = true),
StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true)))
}
}
Loading