From ed01ff0d40fbe65b3a239b196a90013119ad3580 Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Wed, 13 Jun 2018 17:57:17 +0200 Subject: [PATCH 1/3] Different config for same test in SQLQueryTestSuite --- .../native/decimalArithmeticOperations.sql | 29 --- ...malOperations.allowPrecisionLoss-false.out | 193 ++++++++++++++++++ ...malOperations.allowPrecisionLoss-true.out} | 148 +------------- .../apache/spark/sql/SQLQueryTestSuite.scala | 62 +++++- 4 files changed, 252 insertions(+), 180 deletions(-) create mode 100644 sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out rename sql/core/src/test/resources/sql-tests/results/typeCoercion/native/{decimalArithmeticOperations.sql.out => decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out} (55%) diff --git a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql index 28a0e20c0f49..79b8eceba8b7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql @@ -54,33 +54,4 @@ select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.1 select 123456789123456789.1234567890 * 1.123456789123456789; select 12345678912345.123456789123 / 0.000000012345678; --- return NULL instead of rounding, according to old Spark versions' behavior -set spark.sql.decimalOperations.allowPrecisionLoss=false; - --- test decimal operations -select id, a+b, a-b, a*b, a/b from decimals_test order by id; - --- test operations between decimals and constants -select id, a*10, b/10 from decimals_test order by id; - --- test operations on constants -select 10.3 * 3.0; -select 10.3000 * 3.0; -select 10.30000 * 30.0; -select 10.300000000000000000 * 3.000000000000000000; -select 10.300000000000000000 * 3.0000000000000000000; -select 2.35E10 * 1.0; - --- arithmetic operations causing an overflow return NULL -select (5e36 + 0.1) + 5e36; -select (-4e36 - 0.1) - 7e36; -select 12345678901234567890.0 * 12345678901234567890.0; -select 1e35 / 0.1; -select 1.2345678901234567890E30 * 1.2345678901234567890E25; - --- arithmetic operations causing a precision loss return NULL -select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345; -select 123456789123456789.1234567890 * 1.123456789123456789; -select 12345678912345.123456789123 / 0.000000012345678; - drop table decimals_test; diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out new file mode 100644 index 000000000000..1c5b6a1af09f --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out @@ -0,0 +1,193 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 23 + + +-- !query 0 +CREATE TEMPORARY VIEW t AS SELECT 1.0 as a, 0.0 as b +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +select a / b from t +-- !query 1 schema +struct<(CAST(a AS DECIMAL(2,1)) / CAST(b AS DECIMAL(2,1))):decimal(8,6)> +-- !query 1 output +NULL + + +-- !query 2 +select a % b from t +-- !query 2 schema +struct<(CAST(a AS DECIMAL(2,1)) % CAST(b AS DECIMAL(2,1))):decimal(1,1)> +-- !query 2 output +NULL + + +-- !query 3 +select pmod(a, b) from t +-- !query 3 schema +struct +-- !query 3 output +NULL + + +-- !query 4 +create table decimals_test(id int, a decimal(38,18), b decimal(38,18)) using parquet +-- !query 4 schema +struct<> +-- !query 4 output + + + +-- !query 5 +insert into decimals_test values(1, 100.0, 999.0), (2, 12345.123, 12345.123), + (3, 0.1234567891011, 1234.1), (4, 123456789123456789.0, 1.123456789123456789) +-- !query 5 schema +struct<> +-- !query 5 output + + + +-- !query 6 +select id, a+b, a-b, a*b, a/b from decimals_test order by id +-- !query 6 schema +struct +-- !query 6 output +1 1099 -899 NULL 0.1001001001001001 +2 24690.246 0 NULL 1 +3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123 +4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436 + + +-- !query 7 +select id, a*10, b/10 from decimals_test order by id +-- !query 7 schema +struct +-- !query 7 output +1 1000 99.9 +2 123451.23 1234.5123 +3 1.234567891011 123.41 +4 1234567891234567890 0.1123456789123456789 + + +-- !query 8 +select 10.3 * 3.0 +-- !query 8 schema +struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)> +-- !query 8 output +30.9 + + +-- !query 9 +select 10.3000 * 3.0 +-- !query 9 schema +struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)> +-- !query 9 output +30.9 + + +-- !query 10 +select 10.30000 * 30.0 +-- !query 10 schema +struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)> +-- !query 10 output +309 + + +-- !query 11 +select 10.300000000000000000 * 3.000000000000000000 +-- !query 11 schema +struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)> +-- !query 11 output +30.9 + + +-- !query 12 +select 10.300000000000000000 * 3.0000000000000000000 +-- !query 12 schema +struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)> +-- !query 12 output +NULL + + +-- !query 13 +select 2.35E10 * 1.0 +-- !query 13 schema +struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)> +-- !query 13 output +23500000000 + + +-- !query 14 +select (5e36 + 0.1) + 5e36 +-- !query 14 schema +struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)> +-- !query 14 output +NULL + + +-- !query 15 +select (-4e36 - 0.1) - 7e36 +-- !query 15 schema +struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)> +-- !query 15 output +NULL + + +-- !query 16 +select 12345678901234567890.0 * 12345678901234567890.0 +-- !query 16 schema +struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)> +-- !query 16 output +NULL + + +-- !query 17 +select 1e35 / 0.1 +-- !query 17 schema +struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)> +-- !query 17 output +NULL + + +-- !query 18 +select 1.2345678901234567890E30 * 1.2345678901234567890E25 +-- !query 18 schema +struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)> +-- !query 18 output +NULL + + +-- !query 19 +select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345 +-- !query 19 schema +struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)> +-- !query 19 output +NULL + + +-- !query 20 +select 123456789123456789.1234567890 * 1.123456789123456789 +-- !query 20 schema +struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)> +-- !query 20 output +NULL + + +-- !query 21 +select 12345678912345.123456789123 / 0.000000012345678 +-- !query 21 schema +struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)> +-- !query 21 output +NULL + + +-- !query 22 +drop table decimals_test +-- !query 22 schema +struct<> +-- !query 22 output + diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out similarity index 55% rename from sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out rename to sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out index cbf44548b3cc..e24e9b3acabe 100644 --- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 40 +-- Number of queries: 23 -- !query 0 @@ -186,150 +186,8 @@ struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 -- !query 22 -set spark.sql.decimalOperations.allowPrecisionLoss=false --- !query 22 schema -struct --- !query 22 output -spark.sql.decimalOperations.allowPrecisionLoss false - - --- !query 23 -select id, a+b, a-b, a*b, a/b from decimals_test order by id --- !query 23 schema -struct --- !query 23 output -1 1099 -899 NULL 0.1001001001001001 -2 24690.246 0 NULL 1 -3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123 -4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436 - - --- !query 24 -select id, a*10, b/10 from decimals_test order by id --- !query 24 schema -struct --- !query 24 output -1 1000 99.9 -2 123451.23 1234.5123 -3 1.234567891011 123.41 -4 1234567891234567890 0.1123456789123456789 - - --- !query 25 -select 10.3 * 3.0 --- !query 25 schema -struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)> --- !query 25 output -30.9 - - --- !query 26 -select 10.3000 * 3.0 --- !query 26 schema -struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)> --- !query 26 output -30.9 - - --- !query 27 -select 10.30000 * 30.0 --- !query 27 schema -struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)> --- !query 27 output -309 - - --- !query 28 -select 10.300000000000000000 * 3.000000000000000000 --- !query 28 schema -struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)> --- !query 28 output -30.9 - - --- !query 29 -select 10.300000000000000000 * 3.0000000000000000000 --- !query 29 schema -struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)> --- !query 29 output -NULL - - --- !query 30 -select 2.35E10 * 1.0 --- !query 30 schema -struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)> --- !query 30 output -23500000000 - - --- !query 31 -select (5e36 + 0.1) + 5e36 --- !query 31 schema -struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)> --- !query 31 output -NULL - - --- !query 32 -select (-4e36 - 0.1) - 7e36 --- !query 32 schema -struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)> --- !query 32 output -NULL - - --- !query 33 -select 12345678901234567890.0 * 12345678901234567890.0 --- !query 33 schema -struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)> --- !query 33 output -NULL - - --- !query 34 -select 1e35 / 0.1 --- !query 34 schema -struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)> --- !query 34 output -NULL - - --- !query 35 -select 1.2345678901234567890E30 * 1.2345678901234567890E25 --- !query 35 schema -struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)> --- !query 35 output -NULL - - --- !query 36 -select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345 --- !query 36 schema -struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)> --- !query 36 output -NULL - - --- !query 37 -select 123456789123456789.1234567890 * 1.123456789123456789 --- !query 37 schema -struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)> --- !query 37 output -NULL - - --- !query 38 -select 12345678912345.123456789123 / 0.000000012345678 --- !query 38 schema -struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)> --- !query 38 output -NULL - - --- !query 39 drop table decimals_test --- !query 39 schema +-- !query 22 schema struct<> --- !query 39 output +-- !query 22 output diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index beac9699585d..d348f5bde7f1 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} import org.apache.spark.sql.execution.command.{DescribeColumnCommand, DescribeTableCommand} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.sql.types.StructType @@ -104,11 +105,34 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // We should ignore this file from processing. ) + private val configsAllJoinTypes = Seq(Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.defaultValueString), + Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.PREFER_SORTMERGEJOIN.key -> "true"), + Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.PREFER_SORTMERGEJOIN.key -> "false")) -> true + + /** + * Maps a test with the set of configurations it has to run with and a flag indicating whether + * the output must be the same with different configs or it has to be different. + */ + private val testConfigs: Map[String, (Seq[Seq[(String, String)]], Boolean)] = Map( + "typeCoercion/native/decimalArithmeticOperations.sql" -> + (Seq(Seq(SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key -> "true"), + Seq(SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key -> "false")) -> false), + "subquery/in-subquery/in-joins.sql" -> configsAllJoinTypes, + "subquery/in-subquery/not-in-joins.sql" -> configsAllJoinTypes, + "subquery/exists-subquery/exists-joins-and-set-ops.sql" -> configsAllJoinTypes, + "join-empty-relation.sql" -> configsAllJoinTypes, + "natural-join.sql" -> configsAllJoinTypes, + "outer-join.sql" -> configsAllJoinTypes) + // Create all the test cases. listTestCases().foreach(createScalaTestCase) /** A test case. */ - private case class TestCase(name: String, inputFile: String, resultFile: String) + private case class TestCase( + name: String, inputFile: String, resultFile: String, configs: Seq[(String, String)]) /** A single SQL query's output. */ private case class QueryOutput(sql: String, schema: String, output: String) { @@ -150,6 +174,10 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { val localSparkSession = spark.newSession() loadTestData(localSparkSession) + // Execute the list of set operation in order to add the desired configs + val setOperations = testCase.configs.map { case (key, value) => s"set $key=$value" } + setOperations.foreach(localSparkSession.sql) + // Run the SQL queries preparing them for comparison. val outputs: Seq[QueryOutput] = queries.map { sql => val (schema, output) = getNormalizedResult(localSparkSession, sql) @@ -250,11 +278,33 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { } private def listTestCases(): Seq[TestCase] = { - listFilesRecursively(new File(inputFilePath)).map { file => - val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" - val absPath = file.getAbsolutePath - val testCaseName = absPath.stripPrefix(inputFilePath).stripPrefix(File.separator) - TestCase(testCaseName, absPath, resultFile) + listFilesRecursively(new File(inputFilePath)).flatMap { file => + testCases(file.getAbsolutePath) + } + } + + private def testCases(inputPath: String): Seq[TestCase] = { + val baseResultFileName = inputPath.replace(inputFilePath, goldenFilePath) + val testCaseName = inputPath.stripPrefix(inputFilePath).stripPrefix(File.separator) + testConfigs.get(testCaseName) match { + case None => Seq(TestCase(testCaseName, inputPath, s"$baseResultFileName.out", Seq.empty)) + case Some((listOfConfigs, hasSameResults)) => + val allTests = listOfConfigs.map { configs => + val configsSuffix = configs.map { case (key, value) => s"$key-$value" }.mkString("_") + val resultFile = if (hasSameResults) { + s"$baseResultFileName.out" + } else { + s"${baseResultFileName}_$configsSuffix.out" + } + TestCase(s"${testCaseName}_$configsSuffix", inputPath, resultFile, configs) + } + // When regenerating the golden files, if the result is the same for all configs, we need + // to run only one of them. + if (regenerateGoldenFiles && hasSameResults) { + Seq(allTests.head) + } else { + allTests + } } } From 94cbe24a7c341397330589888dbedb2163de42ca Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Wed, 11 Jul 2018 11:20:09 +0200 Subject: [PATCH 2/3] Revert "Different config for same test in SQLQueryTestSuite" This reverts commit ed01ff0d40fbe65b3a239b196a90013119ad3580. --- .../native/decimalArithmeticOperations.sql | 29 +++ ...ut => decimalArithmeticOperations.sql.out} | 148 +++++++++++++- ...malOperations.allowPrecisionLoss-false.out | 193 ------------------ .../apache/spark/sql/SQLQueryTestSuite.scala | 62 +----- 4 files changed, 180 insertions(+), 252 deletions(-) rename sql/core/src/test/resources/sql-tests/results/typeCoercion/native/{decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out => decimalArithmeticOperations.sql.out} (55%) delete mode 100644 sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out diff --git a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql index 79b8eceba8b7..28a0e20c0f49 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql @@ -54,4 +54,33 @@ select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.1 select 123456789123456789.1234567890 * 1.123456789123456789; select 12345678912345.123456789123 / 0.000000012345678; +-- return NULL instead of rounding, according to old Spark versions' behavior +set spark.sql.decimalOperations.allowPrecisionLoss=false; + +-- test decimal operations +select id, a+b, a-b, a*b, a/b from decimals_test order by id; + +-- test operations between decimals and constants +select id, a*10, b/10 from decimals_test order by id; + +-- test operations on constants +select 10.3 * 3.0; +select 10.3000 * 3.0; +select 10.30000 * 30.0; +select 10.300000000000000000 * 3.000000000000000000; +select 10.300000000000000000 * 3.0000000000000000000; +select 2.35E10 * 1.0; + +-- arithmetic operations causing an overflow return NULL +select (5e36 + 0.1) + 5e36; +select (-4e36 - 0.1) - 7e36; +select 12345678901234567890.0 * 12345678901234567890.0; +select 1e35 / 0.1; +select 1.2345678901234567890E30 * 1.2345678901234567890E25; + +-- arithmetic operations causing a precision loss return NULL +select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345; +select 123456789123456789.1234567890 * 1.123456789123456789; +select 12345678912345.123456789123 / 0.000000012345678; + drop table decimals_test; diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out similarity index 55% rename from sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out rename to sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out index e24e9b3acabe..cbf44548b3cc 100644 --- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-true.out +++ b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 23 +-- Number of queries: 40 -- !query 0 @@ -186,8 +186,150 @@ struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 -- !query 22 -drop table decimals_test +set spark.sql.decimalOperations.allowPrecisionLoss=false -- !query 22 schema -struct<> +struct -- !query 22 output +spark.sql.decimalOperations.allowPrecisionLoss false + + +-- !query 23 +select id, a+b, a-b, a*b, a/b from decimals_test order by id +-- !query 23 schema +struct +-- !query 23 output +1 1099 -899 NULL 0.1001001001001001 +2 24690.246 0 NULL 1 +3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123 +4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436 + + +-- !query 24 +select id, a*10, b/10 from decimals_test order by id +-- !query 24 schema +struct +-- !query 24 output +1 1000 99.9 +2 123451.23 1234.5123 +3 1.234567891011 123.41 +4 1234567891234567890 0.1123456789123456789 + + +-- !query 25 +select 10.3 * 3.0 +-- !query 25 schema +struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)> +-- !query 25 output +30.9 + + +-- !query 26 +select 10.3000 * 3.0 +-- !query 26 schema +struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)> +-- !query 26 output +30.9 + + +-- !query 27 +select 10.30000 * 30.0 +-- !query 27 schema +struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)> +-- !query 27 output +309 + + +-- !query 28 +select 10.300000000000000000 * 3.000000000000000000 +-- !query 28 schema +struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)> +-- !query 28 output +30.9 + + +-- !query 29 +select 10.300000000000000000 * 3.0000000000000000000 +-- !query 29 schema +struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)> +-- !query 29 output +NULL + + +-- !query 30 +select 2.35E10 * 1.0 +-- !query 30 schema +struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)> +-- !query 30 output +23500000000 + + +-- !query 31 +select (5e36 + 0.1) + 5e36 +-- !query 31 schema +struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)> +-- !query 31 output +NULL + + +-- !query 32 +select (-4e36 - 0.1) - 7e36 +-- !query 32 schema +struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)> +-- !query 32 output +NULL + + +-- !query 33 +select 12345678901234567890.0 * 12345678901234567890.0 +-- !query 33 schema +struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)> +-- !query 33 output +NULL + + +-- !query 34 +select 1e35 / 0.1 +-- !query 34 schema +struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)> +-- !query 34 output +NULL + + +-- !query 35 +select 1.2345678901234567890E30 * 1.2345678901234567890E25 +-- !query 35 schema +struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)> +-- !query 35 output +NULL + + +-- !query 36 +select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345 +-- !query 36 schema +struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)> +-- !query 36 output +NULL + + +-- !query 37 +select 123456789123456789.1234567890 * 1.123456789123456789 +-- !query 37 schema +struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)> +-- !query 37 output +NULL + + +-- !query 38 +select 12345678912345.123456789123 / 0.000000012345678 +-- !query 38 schema +struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)> +-- !query 38 output +NULL + + +-- !query 39 +drop table decimals_test +-- !query 39 schema +struct<> +-- !query 39 output diff --git a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out deleted file mode 100644 index 1c5b6a1af09f..000000000000 --- a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql_spark.sql.decimalOperations.allowPrecisionLoss-false.out +++ /dev/null @@ -1,193 +0,0 @@ --- Automatically generated by SQLQueryTestSuite --- Number of queries: 23 - - --- !query 0 -CREATE TEMPORARY VIEW t AS SELECT 1.0 as a, 0.0 as b --- !query 0 schema -struct<> --- !query 0 output - - - --- !query 1 -select a / b from t --- !query 1 schema -struct<(CAST(a AS DECIMAL(2,1)) / CAST(b AS DECIMAL(2,1))):decimal(8,6)> --- !query 1 output -NULL - - --- !query 2 -select a % b from t --- !query 2 schema -struct<(CAST(a AS DECIMAL(2,1)) % CAST(b AS DECIMAL(2,1))):decimal(1,1)> --- !query 2 output -NULL - - --- !query 3 -select pmod(a, b) from t --- !query 3 schema -struct --- !query 3 output -NULL - - --- !query 4 -create table decimals_test(id int, a decimal(38,18), b decimal(38,18)) using parquet --- !query 4 schema -struct<> --- !query 4 output - - - --- !query 5 -insert into decimals_test values(1, 100.0, 999.0), (2, 12345.123, 12345.123), - (3, 0.1234567891011, 1234.1), (4, 123456789123456789.0, 1.123456789123456789) --- !query 5 schema -struct<> --- !query 5 output - - - --- !query 6 -select id, a+b, a-b, a*b, a/b from decimals_test order by id --- !query 6 schema -struct --- !query 6 output -1 1099 -899 NULL 0.1001001001001001 -2 24690.246 0 NULL 1 -3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123 -4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436 - - --- !query 7 -select id, a*10, b/10 from decimals_test order by id --- !query 7 schema -struct --- !query 7 output -1 1000 99.9 -2 123451.23 1234.5123 -3 1.234567891011 123.41 -4 1234567891234567890 0.1123456789123456789 - - --- !query 8 -select 10.3 * 3.0 --- !query 8 schema -struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)> --- !query 8 output -30.9 - - --- !query 9 -select 10.3000 * 3.0 --- !query 9 schema -struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)> --- !query 9 output -30.9 - - --- !query 10 -select 10.30000 * 30.0 --- !query 10 schema -struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)> --- !query 10 output -309 - - --- !query 11 -select 10.300000000000000000 * 3.000000000000000000 --- !query 11 schema -struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)> --- !query 11 output -30.9 - - --- !query 12 -select 10.300000000000000000 * 3.0000000000000000000 --- !query 12 schema -struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)> --- !query 12 output -NULL - - --- !query 13 -select 2.35E10 * 1.0 --- !query 13 schema -struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)> --- !query 13 output -23500000000 - - --- !query 14 -select (5e36 + 0.1) + 5e36 --- !query 14 schema -struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)> --- !query 14 output -NULL - - --- !query 15 -select (-4e36 - 0.1) - 7e36 --- !query 15 schema -struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)> --- !query 15 output -NULL - - --- !query 16 -select 12345678901234567890.0 * 12345678901234567890.0 --- !query 16 schema -struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)> --- !query 16 output -NULL - - --- !query 17 -select 1e35 / 0.1 --- !query 17 schema -struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)> --- !query 17 output -NULL - - --- !query 18 -select 1.2345678901234567890E30 * 1.2345678901234567890E25 --- !query 18 schema -struct<(CAST(1.2345678901234567890E+30 AS DECIMAL(25,-6)) * CAST(1.2345678901234567890E+25 AS DECIMAL(25,-6))):decimal(38,-17)> --- !query 18 output -NULL - - --- !query 19 -select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345 --- !query 19 schema -struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)> --- !query 19 output -NULL - - --- !query 20 -select 123456789123456789.1234567890 * 1.123456789123456789 --- !query 20 schema -struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)> --- !query 20 output -NULL - - --- !query 21 -select 12345678912345.123456789123 / 0.000000012345678 --- !query 21 schema -struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)> --- !query 21 output -NULL - - --- !query 22 -drop table decimals_test --- !query 22 schema -struct<> --- !query 22 output - diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index d348f5bde7f1..beac9699585d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -27,7 +27,6 @@ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.rules.RuleExecutor import org.apache.spark.sql.catalyst.util.{fileToString, stringToFile} import org.apache.spark.sql.execution.command.{DescribeColumnCommand, DescribeTableCommand} -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.sql.types.StructType @@ -105,34 +104,11 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // We should ignore this file from processing. ) - private val configsAllJoinTypes = Seq(Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.defaultValueString), - Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "true"), - Seq(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "false")) -> true - - /** - * Maps a test with the set of configurations it has to run with and a flag indicating whether - * the output must be the same with different configs or it has to be different. - */ - private val testConfigs: Map[String, (Seq[Seq[(String, String)]], Boolean)] = Map( - "typeCoercion/native/decimalArithmeticOperations.sql" -> - (Seq(Seq(SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key -> "true"), - Seq(SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key -> "false")) -> false), - "subquery/in-subquery/in-joins.sql" -> configsAllJoinTypes, - "subquery/in-subquery/not-in-joins.sql" -> configsAllJoinTypes, - "subquery/exists-subquery/exists-joins-and-set-ops.sql" -> configsAllJoinTypes, - "join-empty-relation.sql" -> configsAllJoinTypes, - "natural-join.sql" -> configsAllJoinTypes, - "outer-join.sql" -> configsAllJoinTypes) - // Create all the test cases. listTestCases().foreach(createScalaTestCase) /** A test case. */ - private case class TestCase( - name: String, inputFile: String, resultFile: String, configs: Seq[(String, String)]) + private case class TestCase(name: String, inputFile: String, resultFile: String) /** A single SQL query's output. */ private case class QueryOutput(sql: String, schema: String, output: String) { @@ -174,10 +150,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { val localSparkSession = spark.newSession() loadTestData(localSparkSession) - // Execute the list of set operation in order to add the desired configs - val setOperations = testCase.configs.map { case (key, value) => s"set $key=$value" } - setOperations.foreach(localSparkSession.sql) - // Run the SQL queries preparing them for comparison. val outputs: Seq[QueryOutput] = queries.map { sql => val (schema, output) = getNormalizedResult(localSparkSession, sql) @@ -278,33 +250,11 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { } private def listTestCases(): Seq[TestCase] = { - listFilesRecursively(new File(inputFilePath)).flatMap { file => - testCases(file.getAbsolutePath) - } - } - - private def testCases(inputPath: String): Seq[TestCase] = { - val baseResultFileName = inputPath.replace(inputFilePath, goldenFilePath) - val testCaseName = inputPath.stripPrefix(inputFilePath).stripPrefix(File.separator) - testConfigs.get(testCaseName) match { - case None => Seq(TestCase(testCaseName, inputPath, s"$baseResultFileName.out", Seq.empty)) - case Some((listOfConfigs, hasSameResults)) => - val allTests = listOfConfigs.map { configs => - val configsSuffix = configs.map { case (key, value) => s"$key-$value" }.mkString("_") - val resultFile = if (hasSameResults) { - s"$baseResultFileName.out" - } else { - s"${baseResultFileName}_$configsSuffix.out" - } - TestCase(s"${testCaseName}_$configsSuffix", inputPath, resultFile, configs) - } - // When regenerating the golden files, if the result is the same for all configs, we need - // to run only one of them. - if (regenerateGoldenFiles && hasSameResults) { - Seq(allTests.head) - } else { - allTests - } + listFilesRecursively(new File(inputFilePath)).map { file => + val resultFile = file.getAbsolutePath.replace(inputFilePath, goldenFilePath) + ".out" + val absPath = file.getAbsolutePath + val testCaseName = absPath.stripPrefix(inputFilePath).stripPrefix(File.separator) + TestCase(testCaseName, absPath, resultFile) } } From 6f90e63b0ca2b929c2e6e8be6b090c4fe0bb9583 Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Wed, 11 Jul 2018 13:11:21 +0200 Subject: [PATCH 3/3] use matrix of configs in the files --- .../sql-tests/inputs/join-empty-relation.sql | 5 ++ .../sql-tests/inputs/natural-join.sql | 5 ++ .../resources/sql-tests/inputs/outer-join.sql | 5 ++ .../exists-joins-and-set-ops.sql | 4 ++ .../inputs/subquery/in-subquery/in-joins.sql | 4 ++ .../subquery/in-subquery/not-in-joins.sql | 4 ++ .../apache/spark/sql/SQLQueryTestSuite.scala | 53 ++++++++++++++++--- 7 files changed, 74 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/resources/sql-tests/inputs/join-empty-relation.sql b/sql/core/src/test/resources/sql-tests/inputs/join-empty-relation.sql index 8afa3270f4de..2e6a5f362a8f 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/join-empty-relation.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/join-empty-relation.sql @@ -1,3 +1,8 @@ +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false + CREATE TEMPORARY VIEW t1 AS SELECT * FROM VALUES (1) AS GROUPING(a); CREATE TEMPORARY VIEW t2 AS SELECT * FROM VALUES (1) AS GROUPING(a); diff --git a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql index 71a50157b766..e0abeda3eb44 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/natural-join.sql @@ -1,3 +1,8 @@ +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false + create temporary view nt1 as select * from values ("one", 1), ("two", 2), diff --git a/sql/core/src/test/resources/sql-tests/inputs/outer-join.sql b/sql/core/src/test/resources/sql-tests/inputs/outer-join.sql index cdc6c81e1004..ce09c21568f1 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/outer-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/outer-join.sql @@ -1,3 +1,8 @@ +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false + -- SPARK-17099: Incorrect result when HAVING clause is added to group by query CREATE OR REPLACE TEMPORARY VIEW t1 AS SELECT * FROM VALUES (-234), (145), (367), (975), (298) diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/exists-subquery/exists-joins-and-set-ops.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/exists-subquery/exists-joins-and-set-ops.sql index cc4ed64affec..cefc3fe6272a 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/exists-subquery/exists-joins-and-set-ops.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/exists-subquery/exists-joins-and-set-ops.sql @@ -1,5 +1,9 @@ -- Tests EXISTS subquery support. Tests Exists subquery -- used in Joins (Both when joins occurs in outer and suquery blocks) +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false CREATE TEMPORARY VIEW EMP AS SELECT * FROM VALUES (100, "emp 1", date "2005-01-01", 100.00D, 10), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql index 880175fd7add..22f3eafd6a02 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/in-joins.sql @@ -1,5 +1,9 @@ -- A test suite for IN JOINS in parent side, subquery, and both predicate subquery -- It includes correlated cases. +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql index e09b91f18de0..4f8ca8bfb27c 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/subquery/in-subquery/not-in-joins.sql @@ -1,5 +1,9 @@ -- A test suite for not-in-joins in parent side, subquery, and both predicate subquery -- It includes correlated cases. +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false create temporary view t1 as select * from values ("val1a", 6S, 8, 10L, float(15.0), 20D, 20E2, timestamp '2014-04-04 01:00:00.000', date '2014-04-04'), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index beac9699585d..826408c7161e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -54,6 +54,7 @@ import org.apache.spark.sql.types.StructType * The format for input files is simple: * 1. A list of SQL queries separated by semicolon. * 2. Lines starting with -- are treated as comments and ignored. + * 3. Lines starting with --SET are used to run the file with the following set of configs. * * For example: * {{{ @@ -138,18 +139,58 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { private def runTest(testCase: TestCase): Unit = { val input = fileToString(new File(testCase.inputFile)) + val (comments, code) = input.split("\n").partition(_.startsWith("--")) + val configSets = { + val configLines = comments.filter(_.startsWith("--SET")).map(_.substring(5)) + val configs = configLines.map(_.split(",").map { confAndValue => + val (conf, value) = confAndValue.span(_ != '=') + conf.trim -> value.substring(1).trim + }) + // When we are regenerating the golden files we don't need to run all the configs as they + // all need to return the same result + if (regenerateGoldenFiles && configs.nonEmpty) { + configs.take(1) + } else { + configs + } + } // List of SQL queries to run - val queries: Seq[String] = { - val cleaned = input.split("\n").filterNot(_.startsWith("--")).mkString("\n") - // note: this is not a robust way to split queries using semicolon, but works for now. - cleaned.split("(?<=[^\\\\]);").map(_.trim).filter(_ != "").toSeq + // note: this is not a robust way to split queries using semicolon, but works for now. + val queries = code.mkString("\n").split("(?<=[^\\\\]);").map(_.trim).filter(_ != "").toSeq + + if (configSets.isEmpty) { + runQueries(queries, testCase.resultFile, None) + } else { + configSets.foreach { configSet => + try { + runQueries(queries, testCase.resultFile, Some(configSet)) + } catch { + case e: Throwable => + val configs = configSet.map { + case (k, v) => s"$k=$v" + } + logError(s"Error using configs: ${configs.mkString(",")}") + throw e + } + } } + } + private def runQueries( + queries: Seq[String], + resultFileName: String, + configSet: Option[Seq[(String, String)]]): Unit = { // Create a local SparkSession to have stronger isolation between different test cases. // This does not isolate catalog changes. val localSparkSession = spark.newSession() loadTestData(localSparkSession) + if (configSet.isDefined) { + // Execute the list of set operation in order to add the desired configs + val setOperations = configSet.get.map { case (key, value) => s"set $key=$value" } + logInfo(s"Setting configs: ${setOperations.mkString(", ")}") + setOperations.foreach(localSparkSession.sql) + } // Run the SQL queries preparing them for comparison. val outputs: Seq[QueryOutput] = queries.map { sql => val (schema, output) = getNormalizedResult(localSparkSession, sql) @@ -167,7 +208,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { s"-- Number of queries: ${outputs.size}\n\n\n" + outputs.zipWithIndex.map{case (qr, i) => qr.toString(i)}.mkString("\n\n\n") + "\n" } - val resultFile = new File(testCase.resultFile) + val resultFile = new File(resultFileName) val parent = resultFile.getParentFile if (!parent.exists()) { assert(parent.mkdirs(), "Could not create directory: " + parent) @@ -177,7 +218,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext { // Read back the golden file. val expectedOutputs: Seq[QueryOutput] = { - val goldenOutput = fileToString(new File(testCase.resultFile)) + val goldenOutput = fileToString(new File(resultFileName)) val segments = goldenOutput.split("-- !query.+\n") // each query has 3 segments, plus the header