From d167acbe5a6007f2a41c27f552795976512c847e Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 24 Aug 2015 16:09:18 +0900 Subject: [PATCH 1/3] [SPARK-10180] [SQL] JDBCRDD does not process EqualNullSafe filter. --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index e537d631f455..39fa88bd0990 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -275,6 +275,7 @@ private[sql] class JDBCRDD( */ private def compileFilter(f: Filter): String = f match { case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" + case EqualNullSafe(attr, value) => s"$attr <=> ${compileValue(value)}" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}" From f76021d83b5d5bb5c1ac7e3ca5d085359667143d Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 25 Aug 2015 20:53:51 +0900 Subject: [PATCH 2/3] [SPARK-10180] [SQL] Rewirte null-safe equality operator as using isnull and normal equality. --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 39fa88bd0990..0d74dcbdd148 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -275,7 +275,8 @@ private[sql] class JDBCRDD( */ private def compileFilter(f: Filter): String = f match { case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" - case EqualNullSafe(attr, value) => s"$attr <=> ${compileValue(value)}" + case EqualNullSafe(attr, value) => + s"$attr = ${compileValue(value)} OR ($attr IS NULL AND ${compileValue(value)} IS NULL)" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}" From 4fe4b067a976bb396ba34b89f19b10abd548fc6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B6=8C=ED=98=81=EC=A7=84?= Date: Wed, 26 Aug 2015 00:06:03 +0900 Subject: [PATCH 3/3] [SPARK-10180] [SQL] Add some comments and round brackets --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 0d74dcbdd148..4cefc7dee1d8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -275,8 +275,10 @@ private[sql] class JDBCRDD( */ private def compileFilter(f: Filter): String = f match { case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" + // Since the null-safe equality operator is not a standard SQL operator, + // This was written as using is-null and normal equality. case EqualNullSafe(attr, value) => - s"$attr = ${compileValue(value)} OR ($attr IS NULL AND ${compileValue(value)} IS NULL)" + s"($attr = ${compileValue(value)} OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}"