@@ -23,7 +23,6 @@ import scala.language.postfixOps
2323
2424import org .apache .spark .sql .functions ._
2525import org .apache .spark .sql .test .SharedSQLContext
26- import org .apache .spark .sql .types ._
2726
2827
2928class DatasetSuite extends QueryTest with SharedSQLContext {
@@ -490,117 +489,12 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
490489 }
491490 assert(e.getMessage.contains(" cannot resolve 'c' given input columns a, b" ), e.getMessage)
492491 }
493-
494- def testNonNullable [T : Encoder ](name : String , schema : StructType , rows : Row * ): Unit = {
495- test(s " non-nullable field - $name" ) {
496- val rowRDD = sqlContext.sparkContext.parallelize(rows)
497- val ds = sqlContext.createDataFrame(rowRDD, schema).as[T ]
498- val message = intercept[RuntimeException ](ds.collect()).getMessage
499- assert(message.contains(" Null value appeared in non-nullable field" ))
500- }
501- }
502-
503- testNonNullable[ClassData ](
504- " scala.Int" ,
505- StructType (Seq (
506- StructField (" a" , StringType , nullable = true ),
507- StructField (" b" , IntegerType , nullable = false )
508- )),
509- Row (" hello" , 1 : Integer ),
510- Row (" world" , null )
511- )
512-
513- testNonNullable[NestedClassData ](
514- " struct" ,
515- StructType (Seq (
516- StructField (" a" , StructType (Seq (
517- StructField (" a" , StringType , nullable = true ),
518- StructField (" b" , IntegerType , nullable = false )
519- )), nullable = false )
520- )),
521- Row (Row (" hello" , 1 : Integer )),
522- Row (null )
523- )
524-
525- ignore(" non-nullable field in nested struct" ) {
526- testNonNullable[NestedClassData ](
527- " non-nullable field in nested struct" ,
528- StructType (Seq (
529- StructField (" a" , StructType (Seq (
530- StructField (" a" , StringType , nullable = true ),
531- StructField (" b" , IntegerType , nullable = false )
532- )), nullable = false )
533- )),
534- Row (Row (" hello" , 1 : Integer )),
535- Row (Row (" hello" , null ))
536- )
537- }
538-
539- testNonNullable[NestedNonNullableArray ](
540- " array" ,
541- StructType (Seq (
542- StructField (" a" , ArrayType (StructType (Seq (
543- StructField (" a" , StringType , nullable = true ),
544- StructField (" b" , IntegerType , nullable = false )
545- ))), nullable = false )
546- )),
547- Row (Seq (Row (" hello" , 1 ))),
548- Row (null )
549- )
550-
551- ignore(" non-nullable element in nested array" ) {
552- testNonNullable[NestedNonNullableArray ](
553- " non-nullable element in nested array" ,
554- StructType (Seq (
555- StructField (" a" , ArrayType (StructType (Seq (
556- StructField (" a" , StringType , nullable = true ),
557- StructField (" b" , IntegerType , nullable = false )
558- )), containsNull = false ), nullable = false )
559- )),
560- Row (Seq (Row (" hello" , 1 ), null ))
561- )
562- }
563-
564- testNonNullable[NestedNonNullableMap ](
565- " map" ,
566- StructType (Seq (
567- StructField (" a" , MapType (
568- IntegerType ,
569- StructType (Seq (
570- StructField (" a" , StringType , nullable = true ),
571- StructField (" b" , IntegerType , nullable = false )
572- ))
573- ), nullable = false )
574- )),
575- Row (Map (1 -> Row (" hello" , 1 ))),
576- Row (null )
577- )
578-
579- ignore(" non-nullable value in nested map" ) {
580- testNonNullable[NestedNonNullableMap ](
581- " non-nullable value in nested map" ,
582- StructType (Seq (
583- StructField (" a" , MapType (
584- IntegerType ,
585- StructType (Seq (
586- StructField (" a" , StringType , nullable = true ),
587- StructField (" b" , IntegerType , nullable = false )
588- ))
589- ), nullable = false )
590- )),
591- Row (Map (1 -> Row (" hello" , 1 ), 2 -> null ))
592- )
593- }
594492}
595493
596494case class ClassData (a : String , b : Int )
597495case class ClassData2 (c : String , d : Int )
598496case class ClassNullableData (a : String , b : Integer )
599497
600- case class NestedClassData (a : ClassData )
601- case class NestedNonNullableArray (a : Array [ClassData ])
602- case class NestedNonNullableMap (a : scala.collection.Map [Int , ClassData ])
603-
604498/**
605499 * A class used to test serialization using encoders. This class throws exceptions when using
606500 * Java serialization -- so the only way it can be "serialized" is through our encoders.
0 commit comments