Skip to content

Commit d507cfc

Browse files
committed
addressed comments
1 parent d9e5b4f commit d507cfc

File tree

3 files changed

+11
-11
lines changed

3 files changed

+11
-11
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,12 +120,11 @@ object RowEncoder {
120120
inputObject :: Nil)
121121

122122
case t @ ArrayType(et, cn) =>
123-
val cls = inputObject.dataType.asInstanceOf[ObjectType].cls
124123
et match {
125124
case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType =>
126125
StaticInvoke(
127126
classOf[ArrayData],
128-
ObjectType(classOf[ArrayData]),
127+
t,
129128
"toArrayData",
130129
inputObject :: Nil)
131130
case _ => MapObjects(

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ class RowEncoderSuite extends SparkFunSuite {
191191
assert(encoder.serializer.head.nullable == false)
192192
}
193193

194-
test("RowEncoder should support a primitive array") {
194+
test("RowEncoder should support primitive arrays") {
195195
val schema = new StructType()
196196
.add("booleanPrimitiveArray", ArrayType(BooleanType, false))
197197
.add("bytePrimitiveArray", ArrayType(ByteType, false))

sql/core/src/test/scala/org/apache/spark/sql/execution/benchmark/PrimitiveArrayBenchmark.scala

100755100644
Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import org.apache.spark.util.Benchmark
2727
* Benchmark [[PrimitiveArray]] for DataFrame and Dataset program using primitive array
2828
* To run this:
2929
* 1. replace ignore(...) with test(...)
30-
* 2. build/sbt "sql/test-only *benchmark.PrimitiveArrayDataBenchmark"
30+
* 2. build/sbt "sql/test-only *benchmark.PrimitiveArrayBenchmark"
3131
*
3232
* Benchmarks in this file are skipped in normal builds.
3333
*/
@@ -41,22 +41,23 @@ class PrimitiveArrayBenchmark extends BenchmarkBase {
4141
val sc = sparkSession.sparkContext
4242
val primitiveIntArray = Array.fill[Int](count)(65535)
4343
val dsInt = sc.parallelize(Seq(primitiveIntArray), 1).toDS
44-
dsInt.count
44+
dsInt.count // force to build dataset
4545
val intArray = { i: Int =>
4646
var n = 0
47+
var len = 0
4748
while (n < iters) {
48-
dsInt.map(e => e).collect
49+
len += dsInt.map(e => e).queryExecution.toRdd.collect.length
4950
n += 1
5051
}
5152
}
5253
val primitiveDoubleArray = Array.fill[Double](count)(65535.0)
5354
val dsDouble = sc.parallelize(Seq(primitiveDoubleArray), 1).toDS
54-
dsDouble.count
55+
dsDouble.count // force to build dataset
5556
val doubleArray = { i: Int =>
5657
var n = 0
57-
var sum = 0L
58+
var len = 0
5859
while (n < iters) {
59-
dsDouble.map(e => e).collect
60+
len += dsDouble.map(e => e).queryExecution.toRdd.collect.length
6061
n += 1
6162
}
6263
}
@@ -70,8 +71,8 @@ class PrimitiveArrayBenchmark extends BenchmarkBase {
7071
Intel Xeon E3-12xx v2 (Ivy Bridge)
7172
Write an array in Dataset: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative
7273
------------------------------------------------------------------------------------------------
73-
Int 280 / 302 30.0 33.4 1.0X
74-
Double 503 / 519 16.7 60.0 0.6X
74+
Int 352 / 401 23.8 42.0 1.0X
75+
Double 821 / 885 10.2 97.9 0.4X
7576
*/
7677
}
7778

0 commit comments

Comments
 (0)