Skip to content

Commit d540b86

Browse files
committed
Reverts BoundReference changes
1 parent 05c36e5 commit d540b86

File tree

2 files changed

+4
-129
lines changed

2 files changed

+4
-129
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/BoundAttribute.scala

Lines changed: 4 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -69,29 +69,10 @@ case class BoundReference(ordinal: Int, dataType: DataType, nullable: Boolean)
6969
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
7070
val javaType = ctx.javaType(dataType)
7171
val value = ctx.getValue(ctx.INPUT_ROW, dataType, ordinal.toString)
72-
73-
if (nullable) {
74-
s"""
75-
boolean ${ev.isNull} = ${ctx.INPUT_ROW}.isNullAt($ordinal);
76-
$javaType ${ev.value} = ${ev.isNull} ? ${ctx.defaultValue(dataType)} : ($value);
77-
"""
78-
} else {
79-
s"""
80-
boolean ${ev.isNull} = ${ctx.INPUT_ROW}.isNullAt($ordinal);
81-
$javaType ${ev.value};
82-
if (!${ev.isNull}) {
83-
${ev.value} = ($value);
84-
} else {
85-
throw new RuntimeException(
86-
"Null value appeared in non-nullable field: " +
87-
"ordinal=$ordinal, dataType=${dataType.simpleString}. " +
88-
"If the schema is inferred from a Scala tuple/case class, or a Java bean, " +
89-
"please try to use scala.Option[_] or other nullable types " +
90-
"(e.g. java.lang.Integer instead of int/scala.Int)."
91-
);
92-
}
93-
"""
94-
}
72+
s"""
73+
boolean ${ev.isNull} = ${ctx.INPUT_ROW}.isNullAt($ordinal);
74+
$javaType ${ev.value} = ${ev.isNull} ? ${ctx.defaultValue(dataType)} : ($value);
75+
"""
9576
}
9677
}
9778

sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala

Lines changed: 0 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import scala.language.postfixOps
2323

2424
import org.apache.spark.sql.functions._
2525
import org.apache.spark.sql.test.SharedSQLContext
26-
import org.apache.spark.sql.types._
2726

2827

2928
class DatasetSuite extends QueryTest with SharedSQLContext {
@@ -490,117 +489,12 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
490489
}
491490
assert(e.getMessage.contains("cannot resolve 'c' given input columns a, b"), e.getMessage)
492491
}
493-
494-
def testNonNullable[T: Encoder](name: String, schema: StructType, rows: Row*): Unit = {
495-
test(s"non-nullable field - $name") {
496-
val rowRDD = sqlContext.sparkContext.parallelize(rows)
497-
val ds = sqlContext.createDataFrame(rowRDD, schema).as[T]
498-
val message = intercept[RuntimeException](ds.collect()).getMessage
499-
assert(message.contains("Null value appeared in non-nullable field"))
500-
}
501-
}
502-
503-
testNonNullable[ClassData](
504-
"scala.Int",
505-
StructType(Seq(
506-
StructField("a", StringType, nullable = true),
507-
StructField("b", IntegerType, nullable = false)
508-
)),
509-
Row("hello", 1: Integer),
510-
Row("world", null)
511-
)
512-
513-
testNonNullable[NestedClassData](
514-
"struct",
515-
StructType(Seq(
516-
StructField("a", StructType(Seq(
517-
StructField("a", StringType, nullable = true),
518-
StructField("b", IntegerType, nullable = false)
519-
)), nullable = false)
520-
)),
521-
Row(Row("hello", 1: Integer)),
522-
Row(null)
523-
)
524-
525-
ignore("non-nullable field in nested struct") {
526-
testNonNullable[NestedClassData](
527-
"non-nullable field in nested struct",
528-
StructType(Seq(
529-
StructField("a", StructType(Seq(
530-
StructField("a", StringType, nullable = true),
531-
StructField("b", IntegerType, nullable = false)
532-
)), nullable = false)
533-
)),
534-
Row(Row("hello", 1: Integer)),
535-
Row(Row("hello", null))
536-
)
537-
}
538-
539-
testNonNullable[NestedNonNullableArray](
540-
"array",
541-
StructType(Seq(
542-
StructField("a", ArrayType(StructType(Seq(
543-
StructField("a", StringType, nullable = true),
544-
StructField("b", IntegerType, nullable = false)
545-
))), nullable = false)
546-
)),
547-
Row(Seq(Row("hello", 1))),
548-
Row(null)
549-
)
550-
551-
ignore("non-nullable element in nested array") {
552-
testNonNullable[NestedNonNullableArray](
553-
"non-nullable element in nested array",
554-
StructType(Seq(
555-
StructField("a", ArrayType(StructType(Seq(
556-
StructField("a", StringType, nullable = true),
557-
StructField("b", IntegerType, nullable = false)
558-
)), containsNull = false), nullable = false)
559-
)),
560-
Row(Seq(Row("hello", 1), null))
561-
)
562-
}
563-
564-
testNonNullable[NestedNonNullableMap](
565-
"map",
566-
StructType(Seq(
567-
StructField("a", MapType(
568-
IntegerType,
569-
StructType(Seq(
570-
StructField("a", StringType, nullable = true),
571-
StructField("b", IntegerType, nullable = false)
572-
))
573-
), nullable = false)
574-
)),
575-
Row(Map(1 -> Row("hello", 1))),
576-
Row(null)
577-
)
578-
579-
ignore("non-nullable value in nested map") {
580-
testNonNullable[NestedNonNullableMap](
581-
"non-nullable value in nested map",
582-
StructType(Seq(
583-
StructField("a", MapType(
584-
IntegerType,
585-
StructType(Seq(
586-
StructField("a", StringType, nullable = true),
587-
StructField("b", IntegerType, nullable = false)
588-
))
589-
), nullable = false)
590-
)),
591-
Row(Map(1 -> Row("hello", 1), 2 -> null))
592-
)
593-
}
594492
}
595493

596494
case class ClassData(a: String, b: Int)
597495
case class ClassData2(c: String, d: Int)
598496
case class ClassNullableData(a: String, b: Integer)
599497

600-
case class NestedClassData(a: ClassData)
601-
case class NestedNonNullableArray(a: Array[ClassData])
602-
case class NestedNonNullableMap(a: scala.collection.Map[Int, ClassData])
603-
604498
/**
605499
* A class used to test serialization using encoders. This class throws exceptions when using
606500
* Java serialization -- so the only way it can be "serialized" is through our encoders.

0 commit comments

Comments
 (0)