@@ -61,22 +61,6 @@ case class RepeatedData(
6161 mapFieldNull : scala.collection.Map [Int , java.lang.Long ],
6262 structField : PrimitiveData )
6363
64- case class Nested (i : Option [Int ], s : String )
65-
66- case class Data (
67- array : Seq [Int ],
68- arrayContainsNull : Seq [Option [Int ]],
69- map : scala.collection.Map [Int , Long ],
70- mapContainsNul : scala.collection.Map [Int , Option [Long ]],
71- nested : Nested )
72-
73- case class ComplexData (
74- arrayField : Seq [Int ],
75- arrayFieldContainsNull : Seq [Option [Int ]],
76- mapField : scala.collection.Map [Int , Long ],
77- mapFieldContainsNull : scala.collection.Map [Int , Option [Long ]],
78- dataField : Data )
79-
8064case class SpecificCollection (l : List [Int ])
8165
8266/** For testing Kryo serialization based encoder. */
@@ -256,34 +240,28 @@ class ExpressionEncoderSuite extends SparkFunSuite {
256240 ExpressionEncoder .tuple(intEnc, ExpressionEncoder .tuple(intEnc, longEnc))
257241 }
258242
259- test(" Option in Map and Array" ) {
260- val data = ComplexData (
261- Seq (1 , 2 , 3 ),
262- Seq (Some (1 ), Some (2 ), None ),
263- Map (1 -> 10L , 2 -> 20L ),
264- Map (1 -> Some (10L ), 2 -> Some (20L ), 3 -> None ),
265- Data (
266- Seq (10 , 20 , 30 ),
267- Seq (Some (10 ), Some (20 ), None ),
268- Map (10 -> 100L , 20 -> 200L ),
269- Map (10 -> Some (100L ), 20 -> Some (200L ), 30 -> None ),
270- Nested (None , " abc" )))
271-
272- val schema = ScalaReflection .schemaFor[ComplexData ].dataType.asInstanceOf [StructType ]
243+ test(" Option in Array" ) {
244+ val data = (Seq (Some (1 ), Some (2 ), None ), " Option in array" )
245+
246+ val schema = ScalaReflection .schemaFor[Tuple2 [Seq [Option [Int ]], String ]]
247+ .dataType.asInstanceOf [StructType ]
248+ val attributeSeq = schema.toAttributes
249+ val dataEncoder = encoderFor[Tuple2 [Seq [Option [Int ]], String ]]
250+ val boundEncoder = dataEncoder.resolve(attributeSeq, outers).bind(attributeSeq)
251+ assert(boundEncoder.fromRow(boundEncoder.toRow(data)) === (
252+ Seq (Some (1 ), Some (2 ), null ), " Option in array" ))
253+ }
254+
255+ test(" Option in Map" ) {
256+ val data = (Map (1 -> Some (10L ), 2 -> Some (20L ), 3 -> None ), " Option in map" )
257+
258+ val schema = ScalaReflection .schemaFor[Tuple2 [Map [Int , Option [Long ]], String ]]
259+ .dataType.asInstanceOf [StructType ]
273260 val attributeSeq = schema.toAttributes
274- val complexDataEncoder = encoderFor[ComplexData ]
275- val boundEncoder = complexDataEncoder.resolve(attributeSeq, outers).bind(attributeSeq)
276- assert(boundEncoder.fromRow(boundEncoder.toRow(data)) === ComplexData (
277- Seq (1 , 2 , 3 ),
278- Seq (Some (1 ), Some (2 ), null ),
279- Map (1 -> 10L , 2 -> 20L ),
280- Map (1 -> Some (10L ), 2 -> Some (20L ), 3 -> null ),
281- Data (
282- Seq (10 , 20 , 30 ),
283- Seq (Some (10 ), Some (20 ), null ),
284- Map (10 -> 100L , 20 -> 200L ),
285- Map (10 -> Some (100L ), 20 -> Some (200L ), 30 -> null ),
286- Nested (None , " abc" ))))
261+ val dataEncoder = encoderFor[Tuple2 [Map [Int , Option [Long ]], String ]]
262+ val boundEncoder = dataEncoder.resolve(attributeSeq, outers).bind(attributeSeq)
263+ assert(boundEncoder.fromRow(boundEncoder.toRow(data)) === (
264+ Map (1 -> Some (10L ), 2 -> Some (20L ), 3 -> null ), " Option in map" ))
287265 }
288266
289267 test(" nullable of encoder schema" ) {
0 commit comments