Skip to content

[SPARK-2665] [SQL] Add EqualNS & Unit Tests #1570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

[SPARK-2665] [SQL] Add EqualNS & Unit Tests #1570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7af4b0b
Add EqualNS & Unit Tests
chenghao-intel Jul 24, 2014
8e66cdd
Rename the EqualNSTo ==> EqualNullSafe
chenghao-intel Jul 24, 2014
5b2ca88
Add cases into whitelist
chenghao-intel Jul 24, 2014
8d6c789
Remove the test case orc_predicate_pushdown
chenghao-intel Jul 25, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@ trait HiveTypeCoercion {
case e if !e.childrenResolved => e
// No need to change EqualTo operators as that actually makes sense for boolean types.
case e: EqualTo => e
// No need to change the EqualNullSafe operators, too
case e: EqualNullSafe => e
// Otherwise turn them to Byte types so that there exists and ordering.
case p: BinaryComparison
if p.left.dataType == BooleanType && p.right.dataType == BooleanType =>
Expand Down
1 change: 1 addition & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ package object dsl {
def > (other: Expression) = GreaterThan(expr, other)
def >= (other: Expression) = GreaterThanOrEqual(expr, other)
def === (other: Expression) = EqualTo(expr, other)
def <=> (other: Expression) = EqualNullSafe(expr, other)
def !== (other: Expression) = Not(EqualTo(expr, other))

def in(list: Expression*) = In(expr, list)
Expand Down
16 changes: 16 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,22 @@ case class EqualTo(left: Expression, right: Expression) extends BinaryComparison
}
}

case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComparison {
def symbol = "<=>"
override def nullable = false
override def eval(input: Row): Any = {
val l = left.eval(input)
val r = right.eval(input)
if (l == null && r == null) {
true
} else if (l == null || r == null) {
false
} else {
l == r
}
}
}

case class LessThan(left: Expression, right: Expression) extends BinaryComparison {
def symbol = "<"
override def eval(input: Row): Any = c2(input, left, right, _.lt(_, _))
Expand Down
2 changes: 2 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ object NullPropagation extends Rule[LogicalPlan] {
case e @ GetItem(Literal(null, _), _) => Literal(null, e.dataType)
case e @ GetItem(_, Literal(null, _)) => Literal(null, e.dataType)
case e @ GetField(Literal(null, _), _) => Literal(null, e.dataType)
case e @ EqualNullSafe(Literal(null, _), r) => IsNull(r)
case e @ EqualNullSafe(l, Literal(null, _)) => IsNull(l)

// For Coalesce, remove null literals.
case e @ Coalesce(children) =>
Expand Down
10 changes: 9 additions & 1 deletion .../src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,11 +451,13 @@ class ExpressionEvaluationSuite extends FunSuite {
}

test("BinaryComparison") {
val row = new GenericRow(Array[Any](1, 2, 3, null))
val row = new GenericRow(Array[Any](1, 2, 3, null, 3, null))
val c1 = 'a.int.at(0)
val c2 = 'a.int.at(1)
val c3 = 'a.int.at(2)
val c4 = 'a.int.at(3)
val c5 = 'a.int.at(4)
val c6 = 'a.int.at(5)

checkEvaluation(LessThan(c1, c4), null, row)
checkEvaluation(LessThan(c1, c2), true, row)
Expand All @@ -469,6 +471,12 @@ class ExpressionEvaluationSuite extends FunSuite {
checkEvaluation(c1 >= c2, false, row)
checkEvaluation(c1 === c2, false, row)
checkEvaluation(c1 !== c2, true, row)
checkEvaluation(c4 <=> c1, false, row)
checkEvaluation(c1 <=> c4, false, row)
checkEvaluation(c4 <=> c6, true, row)
checkEvaluation(c3 <=> c5, true, row)
checkEvaluation(Literal(true) <=> Literal(null, BooleanType), false, row)
checkEvaluation(Literal(null, BooleanType) <=> Literal(true), false, row)
}

test("StringComparison") {
Expand Down
7 changes: 6 additions & 1 deletion ...atibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,10 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {

// Hive returns the results of describe as plain text. Comments with multiple lines
// introduce extra lines in the Hive results, which make the result comparison fail.
"describe_comment_indent"
"describe_comment_indent",

// Limit clause without a ordering, which causes failure.
"orc_predicate_pushdown"
)

/**
Expand Down Expand Up @@ -503,6 +506,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"join_hive_626",
"join_map_ppr",
"join_nulls",
"join_nullsafe",
"join_rc",
"join_reorder2",
"join_reorder3",
Expand Down Expand Up @@ -734,6 +738,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"udf_double",
"udf_E",
"udf_elt",
"udf_equal",
"udf_exp",
"udf_field",
"udf_find_in_set",
Expand Down
3 changes: 2 additions & 1 deletion sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveContext.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {

protected val primitiveTypes =
Seq(StringType, IntegerType, LongType, DoubleType, FloatType, BooleanType, ByteType,
ShortType, DecimalType, TimestampType)
ShortType, DecimalType, TimestampType, BinaryType)

protected def toHiveString(a: (Any, DataType)): String = a match {
case (struct: Row, StructType(fields)) =>
Expand All @@ -269,6 +269,7 @@ class HiveContext(sc: SparkContext) extends SQLContext(sc) {
}.toSeq.sorted.mkString("{", ",", "}")
case (null, _) => "NULL"
case (t: Timestamp, TimestampType) => new TimestampWritable(t).toString
case (bin: Array[Byte], BinaryType) => new String(bin, "UTF-8")
case (other, tpe) if primitiveTypes contains tpe => other.toString
}

Expand Down
1 change: 1 addition & 0 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -931,6 +931,7 @@ private[hive] object HiveQl {
/* Comparisons */
case Token("=", left :: right:: Nil) => EqualTo(nodeToExpr(left), nodeToExpr(right))
case Token("==", left :: right:: Nil) => EqualTo(nodeToExpr(left), nodeToExpr(right))
case Token("<=>", left :: right:: Nil) => EqualNullSafe(nodeToExpr(left), nodeToExpr(right))
case Token("!=", left :: right:: Nil) => Not(EqualTo(nodeToExpr(left), nodeToExpr(right)))
case Token("<>", left :: right:: Nil) => Not(EqualTo(nodeToExpr(left), nodeToExpr(right)))
case Token(">", left :: right:: Nil) => GreaterThan(nodeToExpr(left), nodeToExpr(right))
Expand Down
1 change: 1 addition & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-0-869726b703f160eabdb7763700b53e60
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-1-5644ab44e5ba9f2941216b8d5dc33a99
View file
Open in desktop
Empty file.
2 changes: 2 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-10-b6de4e85dcc1d1949c7431d39fa1b919
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
NULL 10 10 NULL NULL 10
100 100 100 100 100 100
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-11-3aa243002a5363b84556736ef71613b1
View file
Open in desktop
Empty file.
4 changes: 4 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-12-3cc55b14e8256d2c51361b61986c291e
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
NULL NULL NULL NULL NULL NULL
NULL 10 10 NULL NULL 10
10 NULL NULL 10 10 NULL
100 100 100 100 100 100
12 changes: 12 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-13-69d94e229191e7b9b1a3e7eae46eb993
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
NULL NULL NULL NULL
NULL NULL 10 NULL
NULL NULL 48 NULL
NULL 10 NULL NULL
NULL 10 10 NULL
NULL 10 48 NULL
NULL 35 NULL NULL
NULL 35 10 NULL
NULL 35 48 NULL
10 NULL NULL 10
48 NULL NULL NULL
100 100 100 100
12 changes: 12 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-14-cf9ff6ee72a701a8e2f3e7fb0667903c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
NULL NULL NULL NULL
NULL NULL NULL 35
NULL NULL 10 NULL
NULL NULL 48 NULL
NULL 10 NULL NULL
NULL 10 10 NULL
NULL 10 48 NULL
NULL 35 NULL NULL
NULL 35 10 NULL
NULL 35 48 NULL
10 NULL NULL 10
100 100 100 100
13 changes: 13 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-15-507d0fa6d7ce39e2d9921555cea6f8da
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
NULL NULL NULL NULL
NULL NULL NULL 35
NULL NULL 10 NULL
NULL NULL 48 NULL
NULL 10 NULL NULL
NULL 10 10 NULL
NULL 10 48 NULL
NULL 35 NULL NULL
NULL 35 10 NULL
NULL 35 48 NULL
10 NULL NULL 10
48 NULL NULL NULL
100 100 100 100
11 changes: 11 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-16-1c714fc339304de4db630530e5d1ce97
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
NULL NULL NULL NULL
NULL NULL 10 NULL
NULL NULL 48 NULL
NULL 10 NULL NULL
NULL 10 10 NULL
NULL 10 48 NULL
NULL 35 NULL NULL
NULL 35 10 NULL
NULL 35 48 NULL
10 NULL NULL 10
100 100 100 100
11 changes: 11 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-17-8a4b0dc781a28ad11a0db9805fe03aa8
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
NULL NULL NULL NULL
NULL NULL 10 NULL
NULL NULL 48 NULL
NULL 10 NULL NULL
NULL 10 10 NULL
NULL 10 48 NULL
NULL 35 NULL NULL
NULL 35 10 NULL
NULL 35 48 NULL
10 NULL NULL 10
100 100 100 100
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-18-10b2051e65cac50ee1ea1c138ec192c8
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-19-23ab7ac8229a53d391195be7ca092429
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-2-793e288c9e0971f0bf3f37493f76dc7
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-20-d6fc260320c577eec9a5db0d4135d224
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-21-a60dae725ffc543f805242611d99de4e
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-22-24c80d0f9e3d72c48d947770fa184985
View file
Open in desktop
Empty file.
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-23-3fe6ae20cab3417759dcc654a3a26746
View file
Open in desktop
Empty file.
1 change: 1 addition & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-24-2db30531137611e06fdba478ca7a8412
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0
1 change: 1 addition & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-25-e58b2754e8d9c56a473557a549d0d2b9
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0
1 change: 1 addition & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-26-64cabe5164130a94f387288f37b62d71
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0
42 changes: 42 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-27-e8ed4a1b574a6ca70cbfb3f7b9980aa6
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL 10
NULL NULL NULL 10
NULL NULL NULL 35
NULL NULL NULL 35
NULL NULL NULL 110
NULL NULL NULL 110
NULL NULL NULL 135
NULL NULL NULL 135
NULL 10 NULL NULL
NULL 10 NULL NULL
NULL 10 NULL 10
NULL 10 NULL 35
NULL 10 NULL 110
NULL 10 NULL 135
NULL 35 NULL NULL
NULL 35 NULL NULL
NULL 35 NULL 10
NULL 35 NULL 35
NULL 35 NULL 110
NULL 35 NULL 135
NULL 110 NULL NULL
NULL 110 NULL NULL
NULL 110 NULL 10
NULL 110 NULL 35
NULL 110 NULL 110
NULL 110 NULL 135
NULL 135 NULL NULL
NULL 135 NULL NULL
NULL 135 NULL 10
NULL 135 NULL 35
NULL 135 NULL 110
NULL 135 NULL 135
10 NULL 10 NULL
48 NULL 48 NULL
100 100 100 100
110 NULL 110 NULL
148 NULL 148 NULL
200 200 200 200
14 changes: 14 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-28-5a0c946cd7033857ca99e5fb800f8525
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL 10 NULL 10
NULL 35 NULL 35
NULL 110 NULL 110
NULL 135 NULL 135
10 NULL 10 NULL
48 NULL 48 NULL
100 100 100 100
110 NULL 110 NULL
148 NULL 148 NULL
200 200 200 200
42 changes: 42 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-29-514043c2ddaf6ea8f16a764adc92d1cf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL 10
NULL NULL NULL 10
NULL NULL NULL 35
NULL NULL NULL 35
NULL NULL NULL 110
NULL NULL NULL 110
NULL NULL NULL 135
NULL NULL NULL 135
NULL 10 NULL NULL
NULL 10 NULL NULL
NULL 10 NULL 10
NULL 10 NULL 35
NULL 10 NULL 110
NULL 10 NULL 135
NULL 35 NULL NULL
NULL 35 NULL NULL
NULL 35 NULL 10
NULL 35 NULL 35
NULL 35 NULL 110
NULL 35 NULL 135
NULL 110 NULL NULL
NULL 110 NULL NULL
NULL 110 NULL 10
NULL 110 NULL 35
NULL 110 NULL 110
NULL 110 NULL 135
NULL 135 NULL NULL
NULL 135 NULL NULL
NULL 135 NULL 10
NULL 135 NULL 35
NULL 135 NULL 110
NULL 135 NULL 135
10 NULL 10 NULL
48 NULL 48 NULL
100 100 100 100
110 NULL 110 NULL
148 NULL 148 NULL
200 200 200 200
Empty file added sql/hive/src/test/resources/golden/join_nullsafe-3-ae378fc0f875a21884e58fa35a6d52cd
View file
Open in desktop
Empty file.
42 changes: 42 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-30-fcbf92cb1b85ab01102fbbc6caba9a88
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL 10
NULL NULL NULL 10
NULL NULL NULL 35
NULL NULL NULL 35
NULL NULL NULL 110
NULL NULL NULL 110
NULL NULL NULL 135
NULL NULL NULL 135
NULL 10 NULL NULL
NULL 10 NULL NULL
NULL 10 NULL 10
NULL 10 NULL 35
NULL 10 NULL 110
NULL 10 NULL 135
NULL 35 NULL NULL
NULL 35 NULL NULL
NULL 35 NULL 10
NULL 35 NULL 35
NULL 35 NULL 110
NULL 35 NULL 135
NULL 110 NULL NULL
NULL 110 NULL NULL
NULL 110 NULL 10
NULL 110 NULL 35
NULL 110 NULL 110
NULL 110 NULL 135
NULL 135 NULL NULL
NULL 135 NULL NULL
NULL 135 NULL 10
NULL 135 NULL 35
NULL 135 NULL 110
NULL 135 NULL 135
10 NULL 10 NULL
48 NULL 48 NULL
100 100 100 100
110 NULL 110 NULL
148 NULL 148 NULL
200 200 200 200
42 changes: 42 additions & 0 deletions sql/hive/src/test/resources/golden/join_nullsafe-31-1cb03e1106f79d14f22bc89d386cedcf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL NULL
NULL NULL NULL 10
NULL NULL NULL 10
NULL NULL NULL 35
NULL NULL NULL 35
NULL NULL NULL 110
NULL NULL NULL 110
NULL NULL NULL 135
NULL NULL NULL 135
NULL 10 NULL NULL
NULL 10 NULL NULL
NULL 10 NULL 10
NULL 10 NULL 35
NULL 10 NULL 110
NULL 10 NULL 135
NULL 35 NULL NULL
NULL 35 NULL NULL
NULL 35 NULL 10
NULL 35 NULL 35
NULL 35 NULL 110
NULL 35 NULL 135
NULL 110 NULL NULL
NULL 110 NULL NULL
NULL 110 NULL 10
NULL 110 NULL 35
NULL 110 NULL 110
NULL 110 NULL 135
NULL 135 NULL NULL
NULL 135 NULL NULL
NULL 135 NULL 10
NULL 135 NULL 35
NULL 135 NULL 110
NULL 135 NULL 135
10 NULL 10 NULL
48 NULL 48 NULL
100 100 100 100
110 NULL 110 NULL
148 NULL 148 NULL
200 200 200 200
Loading