From cb1dd0af31fc6835333cea0fdc63cb700d2336fd Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 14 Sep 2015 19:23:26 +0900 Subject: [PATCH 1/5] [SPARK-10180] [SQL] Update query to prevent to return null. --- .../spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index 730d88b024cb..a7bab575af09 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -275,6 +275,11 @@ private[sql] class JDBCRDD( */ private def compileFilter(f: Filter): String = f match { case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" + // Since the null-safe equality operator is not a standard SQL operator, + // This was written as using is-null and normal equality. + case EqualNullSafe(attr, value) => + s"(NOT ($attr <> ${compileValue(value)} OR $attr IS NULL OR " + + s"${compileValue(value)} IS NULL) OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}" From bf0b22c75b1f5084a07daa903d554dafc20d9a6a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 20 Nov 2015 16:53:57 +0900 Subject: [PATCH 2/5] [SPARK-10180][SQL] Add a test for the null-safety comparison operator. --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index ed710689cc67..25d143c83fa1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -180,6 +180,7 @@ class JDBCSuite extends SparkFunSuite with BeforeAndAfter with SharedSQLContext assert(sql("SELECT * FROM foobar WHERE THEID != 2").collect().size === 2) assert(sql("SELECT * FROM foobar WHERE THEID = 1").collect().size === 1) assert(sql("SELECT * FROM foobar WHERE NAME = 'fred'").collect().size === 1) + assert(sql("SELECT * FROM foobar WHERE NAME <=> 'fred'").collect().size === 1) assert(sql("SELECT * FROM foobar WHERE NAME > 'fred'").collect().size === 2) assert(sql("SELECT * FROM foobar WHERE NAME != 'fred'").collect().size === 2) } From ef9350d9d1c14aa69e6b4234bc5d84de96804efe Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 29 Dec 2015 17:42:21 +0900 Subject: [PATCH 3/5] Use != instead of <>. --- .../apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index ec37231837e9..fca0d58714ee 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -318,7 +318,7 @@ private[sql] class JDBCRDD( // Since the null-safe equality operator is not a standard SQL operator, // This was written as using is-null and normal equality. case EqualNullSafe(attr, value) => - s"(NOT ($attr <> ${compileValue(value)} OR $attr IS NULL OR " + + s"(NOT ($attr != ${compileValue(value)} OR $attr IS NULL OR " + s"${compileValue(value)} IS NULL) OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" case Not(EqualTo(attr, value)) => s"$attr != ${compileValue(value)}" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" From bd0b70e1f7e3313f2d31610d306530b2dc4251d3 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 29 Dec 2015 17:47:12 +0900 Subject: [PATCH 4/5] Remove unintentionally added functions of old verions --- .../execution/datasources/jdbc/JDBCRDD.scala | 37 ++----------------- 1 file changed, 3 insertions(+), 34 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index fca0d58714ee..bb53c335182b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -191,6 +191,9 @@ private[sql] object JDBCRDD extends Logging { */ private def compileFilter(f: Filter): String = f match { case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" + case EqualNullSafe(attr, value) => + s"(NOT ($attr != ${compileValue(value)} OR $attr IS NULL OR " + + s"${compileValue(value)} IS NULL) OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" case Not(EqualTo(attr, value)) => s"$attr != ${compileValue(value)}" case LessThan(attr, value) => s"$attr < ${compileValue(value)}" case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" @@ -296,40 +299,6 @@ private[sql] class JDBCRDD( if (sb.length == 0) "1" else sb.substring(1) } - /** - * Converts value to SQL expression. - */ - private def compileValue(value: Any): Any = value match { - case stringValue: String => s"'${escapeSql(stringValue)}'" - case timestampValue: Timestamp => "'" + timestampValue + "'" - case dateValue: Date => "'" + dateValue + "'" - case _ => value - } - - private def escapeSql(value: String): String = - if (value == null) null else StringUtils.replace(value, "'", "''") - - /** - * Turns a single Filter into a String representing a SQL expression. - * Returns null for an unhandled filter. - */ - private def compileFilter(f: Filter): String = f match { - case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" - // Since the null-safe equality operator is not a standard SQL operator, - // This was written as using is-null and normal equality. - case EqualNullSafe(attr, value) => - s"(NOT ($attr != ${compileValue(value)} OR $attr IS NULL OR " + - s"${compileValue(value)} IS NULL) OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" - case Not(EqualTo(attr, value)) => s"$attr != ${compileValue(value)}" - case LessThan(attr, value) => s"$attr < ${compileValue(value)}" - case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" - case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}" - case GreaterThanOrEqual(attr, value) => s"$attr >= ${compileValue(value)}" - case IsNull(attr) => s"$attr IS NULL" - case IsNotNull(attr) => s"$attr IS NOT NULL" - case _ => null - } - /** * `filters`, but as a WHERE clause suitable for injection into a SQL query. */ From a489a336b6ef4364f834262738f877596b67297c Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Sat, 2 Jan 2016 07:52:34 +0900 Subject: [PATCH 5/5] Change the compiled string test. --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 825209cb3519..dae72e8acb5a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -474,7 +474,9 @@ class JDBCSuite extends SparkFunSuite === "(NOT (col1 IN ('mno', 'pqr')))") assert(doCompileFilter(IsNull("col1")) === "col1 IS NULL") assert(doCompileFilter(IsNotNull("col1")) === "col1 IS NOT NULL") - assert(doCompileFilter(And(EqualNullSafe("col0", "abc"), EqualTo("col1", "def"))) === "") + assert(doCompileFilter(And(EqualNullSafe("col0", "abc"), EqualTo("col1", "def"))) + === "((NOT (col0 != 'abc' OR col0 IS NULL OR 'abc' IS NULL) " + + "OR (col0 IS NULL AND 'abc' IS NULL))) AND (col1 = 'def')") } test("Dialect unregister") {