From 66f7999b0a4d669897ddd628ef542756140bb6b0 Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Thu, 27 Oct 2016 11:27:34 -0700 Subject: [PATCH 1/5] [SPARK-18141][SQL] Fix to quote column names in the predicate clause of the JDBC RDD generated sql statement --- .../execution/datasources/jdbc/JDBCRDD.scala | 45 ++++++------ .../datasources/jdbc/JDBCRelation.scala | 4 +- .../org/apache/spark/sql/jdbc/JDBCSuite.scala | 70 ++++++++++++++----- 3 files changed, 81 insertions(+), 38 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index a1e5dfdbf739..c14e3525e7cd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -27,7 +27,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.internal.Logging import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.jdbc.JdbcDialects +import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects} import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ import org.apache.spark.util.CompletionIterator @@ -105,37 +105,40 @@ object JDBCRDD extends Logging { * Turns a single Filter into a String representing a SQL expression. * Returns None for an unhandled filter. */ - def compileFilter(f: Filter): Option[String] = { + def compileFilter(f: Filter, dialect: JdbcDialect): Option[String] = { Option(f match { - case EqualTo(attr, value) => s"$attr = ${compileValue(value)}" + case EqualTo(attr, value) => s"${dialect.quoteIdentifier(attr)} = ${compileValue(value)}" case EqualNullSafe(attr, value) => - s"(NOT ($attr != ${compileValue(value)} OR $attr IS NULL OR " + - s"${compileValue(value)} IS NULL) OR ($attr IS NULL AND ${compileValue(value)} IS NULL))" - case LessThan(attr, value) => s"$attr < ${compileValue(value)}" - case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}" - case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}" - case GreaterThanOrEqual(attr, value) => s"$attr >= ${compileValue(value)}" - case IsNull(attr) => s"$attr IS NULL" - case IsNotNull(attr) => s"$attr IS NOT NULL" - case StringStartsWith(attr, value) => s"${attr} LIKE '${value}%'" - case StringEndsWith(attr, value) => s"${attr} LIKE '%${value}'" - case StringContains(attr, value) => s"${attr} LIKE '%${value}%'" + val col = dialect.quoteIdentifier(attr) + s"(NOT ($col != ${compileValue(value)} OR $col IS NULL OR " + + s"${compileValue(value)} IS NULL) OR ($col IS NULL AND ${compileValue(value)} IS NULL))" + case LessThan(attr, value) => s"${dialect.quoteIdentifier(attr)} < ${compileValue(value)}" + case GreaterThan(attr, value) => s"${dialect.quoteIdentifier(attr)} > ${compileValue(value)}" + case LessThanOrEqual(attr, value) => + s"${dialect.quoteIdentifier(attr)} <= ${compileValue(value)}" + case GreaterThanOrEqual(attr, value) => + s"${dialect.quoteIdentifier(attr)} >= ${compileValue(value)}" + case IsNull(attr) => s"${dialect.quoteIdentifier(attr)} IS NULL" + case IsNotNull(attr) => s"${dialect.quoteIdentifier(attr)} IS NOT NULL" + case StringStartsWith(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '${value}%'" + case StringEndsWith(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '%${value}'" + case StringContains(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '%${value}%'" case In(attr, value) if value.isEmpty => - s"CASE WHEN ${attr} IS NULL THEN NULL ELSE FALSE END" - case In(attr, value) => s"$attr IN (${compileValue(value)})" - case Not(f) => compileFilter(f).map(p => s"(NOT ($p))").getOrElse(null) + s"CASE WHEN ${dialect.quoteIdentifier(attr)} IS NULL THEN NULL ELSE FALSE END" + case In(attr, value) => s"${dialect.quoteIdentifier(attr)} IN (${compileValue(value)})" + case Not(f) => compileFilter(f, dialect).map(p => s"(NOT ($p))").getOrElse(null) case Or(f1, f2) => // We can't compile Or filter unless both sub-filters are compiled successfully. // It applies too for the following And filter. // If we can make sure compileFilter supports all filters, we can remove this check. - val or = Seq(f1, f2).flatMap(compileFilter(_)) + val or = Seq(f1, f2).flatMap(compileFilter(_, dialect)) if (or.size == 2) { or.map(p => s"($p)").mkString(" OR ") } else { null } case And(f1, f2) => - val and = Seq(f1, f2).flatMap(compileFilter(_)) + val and = Seq(f1, f2).flatMap(compileFilter(_, dialect)) if (and.size == 2) { and.map(p => s"($p)").mkString(" AND ") } else { @@ -214,7 +217,9 @@ private[jdbc] class JDBCRDD( * `filters`, but as a WHERE clause suitable for injection into a SQL query. */ private val filterWhereClause: String = - filters.flatMap(JDBCRDD.compileFilter).map(p => s"($p)").mkString(" AND ") + filters + .flatMap(JDBCRDD.compileFilter(_, JdbcDialects.get(url))) + .map(p => s"($p)").mkString(" AND ") /** * A WHERE clause representing both `filters`, if any, and the current partition. diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 672c21c6ac73..0596922dcc97 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -23,9 +23,11 @@ import org.apache.spark.internal.Logging import org.apache.spark.Partition import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession, SQLContext} +import org.apache.spark.sql.jdbc.JdbcDialects import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.StructType + /** * Instructions on how to partition the table among workers. */ @@ -113,7 +115,7 @@ private[sql] case class JDBCRelation( // Check if JDBCRDD.compileFilter can accept input filters override def unhandledFilters(filters: Array[Filter]): Array[Filter] = { - filters.filter(JDBCRDD.compileFilter(_).isEmpty) + filters.filter(JDBCRDD.compileFilter(_, JdbcDialects.get(jdbcOptions.url)).isEmpty) } override def buildScan(requiredColumns: Array[String], filters: Array[Filter]): RDD[Row] = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index b16be457ed5c..9006ffd4332c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -202,6 +202,21 @@ class JDBCSuite extends SparkFunSuite |partitionColumn '"Dept"', lowerBound '1', upperBound '4', numPartitions '4') """.stripMargin.replaceAll("\n", " ")) + conn.prepareStatement( + """create table test."mixedCaseCols" ("Name" TEXT(32), "Id" INTEGER NOT NULL)""") + .executeUpdate() + conn.prepareStatement("""insert into test."mixedCaseCols" values ('fred', 1)""").executeUpdate() + conn.prepareStatement("""insert into test."mixedCaseCols" values ('mary', 2)""").executeUpdate() + conn.prepareStatement("""insert into test."mixedCaseCols" values (null, 3)""").executeUpdate() + conn.commit() + + sql( + s""" + |CREATE TEMPORARY TABLE mixedCaseCols + |USING org.apache.spark.sql.jdbc + |OPTIONS (url '$url', dbtable 'TEST."mixedCaseCols"', user 'testUser', password 'testPass') + """.stripMargin.replaceAll("\n", " ")) + // Untested: IDENTITY, OTHER, UUID, ARRAY, and GEOMETRY types. } @@ -604,30 +619,32 @@ class JDBCSuite extends SparkFunSuite test("compile filters") { val compileFilter = PrivateMethod[Option[String]]('compileFilter) - def doCompileFilter(f: Filter): String = JDBCRDD invokePrivate compileFilter(f) getOrElse("") - assert(doCompileFilter(EqualTo("col0", 3)) === "col0 = 3") - assert(doCompileFilter(Not(EqualTo("col1", "abc"))) === "(NOT (col1 = 'abc'))") + def doCompileFilter(f: Filter): String = + JDBCRDD invokePrivate compileFilter(f, JdbcDialects.get("jdbc:")) getOrElse("") + assert(doCompileFilter(EqualTo("col0", 3)) === """"col0" = 3""") + assert(doCompileFilter(Not(EqualTo("col1", "abc"))) === """(NOT ("col1" = 'abc'))""") assert(doCompileFilter(And(EqualTo("col0", 0), EqualTo("col1", "def"))) - === "(col0 = 0) AND (col1 = 'def')") + === """("col0" = 0) AND ("col1" = 'def')""") assert(doCompileFilter(Or(EqualTo("col0", 2), EqualTo("col1", "ghi"))) - === "(col0 = 2) OR (col1 = 'ghi')") - assert(doCompileFilter(LessThan("col0", 5)) === "col0 < 5") + === """("col0" = 2) OR ("col1" = 'ghi')""") + assert(doCompileFilter(LessThan("col0", 5)) === """"col0" < 5""") assert(doCompileFilter(LessThan("col3", - Timestamp.valueOf("1995-11-21 00:00:00.0"))) === "col3 < '1995-11-21 00:00:00.0'") - assert(doCompileFilter(LessThan("col4", Date.valueOf("1983-08-04"))) === "col4 < '1983-08-04'") - assert(doCompileFilter(LessThanOrEqual("col0", 5)) === "col0 <= 5") - assert(doCompileFilter(GreaterThan("col0", 3)) === "col0 > 3") - assert(doCompileFilter(GreaterThanOrEqual("col0", 3)) === "col0 >= 3") - assert(doCompileFilter(In("col1", Array("jkl"))) === "col1 IN ('jkl')") + Timestamp.valueOf("1995-11-21 00:00:00.0"))) === """"col3" < '1995-11-21 00:00:00.0'""") + assert(doCompileFilter(LessThan("col4", Date.valueOf("1983-08-04"))) + === """"col4" < '1983-08-04'""") + assert(doCompileFilter(LessThanOrEqual("col0", 5)) === """"col0" <= 5""") + assert(doCompileFilter(GreaterThan("col0", 3)) === """"col0" > 3""") + assert(doCompileFilter(GreaterThanOrEqual("col0", 3)) === """"col0" >= 3""") + assert(doCompileFilter(In("col1", Array("jkl"))) === """"col1" IN ('jkl')""") assert(doCompileFilter(In("col1", Array.empty)) === "CASE WHEN col1 IS NULL THEN NULL ELSE FALSE END") assert(doCompileFilter(Not(In("col1", Array("mno", "pqr")))) - === "(NOT (col1 IN ('mno', 'pqr')))") - assert(doCompileFilter(IsNull("col1")) === "col1 IS NULL") - assert(doCompileFilter(IsNotNull("col1")) === "col1 IS NOT NULL") + === """(NOT ("col1" IN ('mno', 'pqr')))""") + assert(doCompileFilter(IsNull("col1")) === """"col1" IS NULL""") + assert(doCompileFilter(IsNotNull("col1")) === """"col1" IS NOT NULL""") assert(doCompileFilter(And(EqualNullSafe("col0", "abc"), EqualTo("col1", "def"))) - === "((NOT (col0 != 'abc' OR col0 IS NULL OR 'abc' IS NULL) " - + "OR (col0 IS NULL AND 'abc' IS NULL))) AND (col1 = 'def')") + === """((NOT ("col0" != 'abc' OR "col0" IS NULL OR 'abc' IS NULL) """ + + """OR ("col0" IS NULL AND 'abc' IS NULL))) AND ("col1" = 'def')""") } test("Dialect unregister") { @@ -824,4 +841,23 @@ class JDBCSuite extends SparkFunSuite val schema = JdbcUtils.schemaString(df.schema, "jdbc:mysql://localhost:3306/temp") assert(schema.contains("`order` TEXT")) } + + test("SPARK-18141: Predicates on quoted column names in the jdbc data source") { + assert(sql("SELECT * FROM mixedCaseCols WHERE Id < 1").collect().size == 0) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id <= 1").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id > 1").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id >= 1").collect().size == 3) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id = 1").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id != 2").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id <=> 2").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE 'fr%'").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE NAME LIKE '%ed'").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE NAME LIKE '%re%'").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NULL").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NOT NULL").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name IN ('mary', 'fred')").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name NOT IN ('fred')").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Id = 1 OR Name = 'mary'").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name = 'mary' AND Id = 2").collect().size == 1) + } } From 5dd575e70e265d34bb62051dbd37004966196823 Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Wed, 2 Nov 2016 20:25:41 -0700 Subject: [PATCH 2/5] Addressed review comments. Simplified code using a nested function --- .../execution/datasources/jdbc/JDBCRDD.scala | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala index c14e3525e7cd..37df283a9e5b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala @@ -106,26 +106,26 @@ object JDBCRDD extends Logging { * Returns None for an unhandled filter. */ def compileFilter(f: Filter, dialect: JdbcDialect): Option[String] = { + def quote(colName: String): String = dialect.quoteIdentifier(colName) + Option(f match { - case EqualTo(attr, value) => s"${dialect.quoteIdentifier(attr)} = ${compileValue(value)}" + case EqualTo(attr, value) => s"${quote(attr)} = ${compileValue(value)}" case EqualNullSafe(attr, value) => - val col = dialect.quoteIdentifier(attr) + val col = quote(attr) s"(NOT ($col != ${compileValue(value)} OR $col IS NULL OR " + s"${compileValue(value)} IS NULL) OR ($col IS NULL AND ${compileValue(value)} IS NULL))" - case LessThan(attr, value) => s"${dialect.quoteIdentifier(attr)} < ${compileValue(value)}" - case GreaterThan(attr, value) => s"${dialect.quoteIdentifier(attr)} > ${compileValue(value)}" - case LessThanOrEqual(attr, value) => - s"${dialect.quoteIdentifier(attr)} <= ${compileValue(value)}" - case GreaterThanOrEqual(attr, value) => - s"${dialect.quoteIdentifier(attr)} >= ${compileValue(value)}" - case IsNull(attr) => s"${dialect.quoteIdentifier(attr)} IS NULL" - case IsNotNull(attr) => s"${dialect.quoteIdentifier(attr)} IS NOT NULL" - case StringStartsWith(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '${value}%'" - case StringEndsWith(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '%${value}'" - case StringContains(attr, value) => s"${dialect.quoteIdentifier(attr)} LIKE '%${value}%'" + case LessThan(attr, value) => s"${quote(attr)} < ${compileValue(value)}" + case GreaterThan(attr, value) => s"${quote(attr)} > ${compileValue(value)}" + case LessThanOrEqual(attr, value) => s"${quote(attr)} <= ${compileValue(value)}" + case GreaterThanOrEqual(attr, value) => s"${quote(attr)} >= ${compileValue(value)}" + case IsNull(attr) => s"${quote(attr)} IS NULL" + case IsNotNull(attr) => s"${quote(attr)} IS NOT NULL" + case StringStartsWith(attr, value) => s"${quote(attr)} LIKE '${value}%'" + case StringEndsWith(attr, value) => s"${quote(attr)} LIKE '%${value}'" + case StringContains(attr, value) => s"${quote(attr)} LIKE '%${value}%'" case In(attr, value) if value.isEmpty => - s"CASE WHEN ${dialect.quoteIdentifier(attr)} IS NULL THEN NULL ELSE FALSE END" - case In(attr, value) => s"${dialect.quoteIdentifier(attr)} IN (${compileValue(value)})" + s"CASE WHEN ${quote(attr)} IS NULL THEN NULL ELSE FALSE END" + case In(attr, value) => s"${quote(attr)} IN (${compileValue(value)})" case Not(f) => compileFilter(f, dialect).map(p => s"(NOT ($p))").getOrElse(null) case Or(f1, f2) => // We can't compile Or filter unless both sub-filters are compiled successfully. From 1765332750071998eba67481532cfd9e82ea2357 Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Thu, 3 Nov 2016 00:30:40 -0700 Subject: [PATCH 3/5] Addressed review comments. Minor fix to test, and remove extra empty line --- .../spark/sql/execution/datasources/jdbc/JDBCRelation.scala | 1 - .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala index 0596922dcc97..6abb27db8531 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala @@ -27,7 +27,6 @@ import org.apache.spark.sql.jdbc.JdbcDialects import org.apache.spark.sql.sources._ import org.apache.spark.sql.types.StructType - /** * Instructions on how to partition the table among workers. */ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index 9006ffd4332c..b6e5e66e5f0f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -851,8 +851,8 @@ class JDBCSuite extends SparkFunSuite assert(sql("SELECT * FROM mixedCaseCols WHERE Id != 2").collect().size == 2) assert(sql("SELECT * FROM mixedCaseCols WHERE Id <=> 2").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE 'fr%'").collect().size == 1) - assert(sql("SELECT * FROM mixedCaseCols WHERE NAME LIKE '%ed'").collect().size == 1) - assert(sql("SELECT * FROM mixedCaseCols WHERE NAME LIKE '%re%'").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE '%ed'").collect().size == 1) + assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE '%re%'").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NULL").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NOT NULL").collect().size == 2) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IN ('mary', 'fred')").collect().size == 2) From 2178e3fe13d4a7d64edbe6f45c2384b911ccde75 Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Mon, 28 Nov 2016 14:38:54 -0800 Subject: [PATCH 4/5] Adding test case for in with empty value list filter using mixed case column --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index b6e5e66e5f0f..e90d6c59a4d7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -637,7 +637,7 @@ class JDBCSuite extends SparkFunSuite assert(doCompileFilter(GreaterThanOrEqual("col0", 3)) === """"col0" >= 3""") assert(doCompileFilter(In("col1", Array("jkl"))) === """"col1" IN ('jkl')""") assert(doCompileFilter(In("col1", Array.empty)) === - "CASE WHEN col1 IS NULL THEN NULL ELSE FALSE END") + """CASE WHEN "col1" IS NULL THEN NULL ELSE FALSE END""") assert(doCompileFilter(Not(In("col1", Array("mno", "pqr")))) === """(NOT ("col1" IN ('mno', 'pqr')))""") assert(doCompileFilter(IsNull("col1")) === """"col1" IS NULL""") @@ -855,6 +855,8 @@ class JDBCSuite extends SparkFunSuite assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE '%re%'").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NULL").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NOT NULL").collect().size == 2) + assert(sql("SELECT * FROM mixedCaseCols") + .filter($"Name".isin(Array[String]() : _*)).collect().size == 0) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IN ('mary', 'fred')").collect().size == 2) assert(sql("SELECT * FROM mixedCaseCols WHERE Name NOT IN ('fred')").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Id = 1 OR Name = 'mary'").collect().size == 2) From f0d731f5c777e919f6b107ae15c3b4abc282d08d Mon Sep 17 00:00:00 2001 From: sureshthalamati Date: Wed, 30 Nov 2016 12:00:36 -0800 Subject: [PATCH 5/5] Addressing review comments. simplified isin test case --- .../src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala index e90d6c59a4d7..af5f01c493e8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala @@ -855,8 +855,7 @@ class JDBCSuite extends SparkFunSuite assert(sql("SELECT * FROM mixedCaseCols WHERE Name LIKE '%re%'").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NULL").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IS NOT NULL").collect().size == 2) - assert(sql("SELECT * FROM mixedCaseCols") - .filter($"Name".isin(Array[String]() : _*)).collect().size == 0) + assert(sql("SELECT * FROM mixedCaseCols").filter($"Name".isin()).collect().size == 0) assert(sql("SELECT * FROM mixedCaseCols WHERE Name IN ('mary', 'fred')").collect().size == 2) assert(sql("SELECT * FROM mixedCaseCols WHERE Name NOT IN ('fred')").collect().size == 1) assert(sql("SELECT * FROM mixedCaseCols WHERE Id = 1 OR Name = 'mary'").collect().size == 2)