From 339e546d59f2887e6fb34065b68460625526db64 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Thu, 5 Aug 2021 16:33:30 +0800 Subject: [PATCH 01/12] [SPARK-36223][SQL][TEST] Cover 3 kinds of join in the TPCDSQueryTestSuite --- .../spark/sql/TPCDSQueryTestSuite.scala | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 3e7f898a228f1..de0c6c73e731a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -24,6 +24,7 @@ import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.catalyst.util.{fileToString, resourceToString, stringToFile} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession +import org.apache.spark.util.Utils /** * End-to-end tests to check TPCDS query results. @@ -97,7 +98,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp """.stripMargin) } - private def runQuery(query: String, goldenFile: File): Unit = { + private def runQuery(query: String, goldenFile: File, needSort: Boolean): Unit = { val (schema, output) = handleExceptions(getNormalizedResult(spark, query)) val queryString = query.trim val outputString = output.mkString("\n").replaceAll("\\s+$", "") @@ -131,16 +132,44 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp } assertResult(expectedSchema, s"Schema did not match\n$queryString") { schema } - assertResult(expectedOutput, s"Result did not match\n$queryString") { outputString } + if (needSort) { + val expectSorted = expectedOutput.split("\n").sorted.map(_.trim) + .mkString("\n").replaceAll("\\s+$", "") + val outputSorted = output.sorted.map(_.trim).mkString("\n").replaceAll("\\s+$", "") + assertResult(expectSorted, s"Result did not match") { outputSorted } + } else { + assertResult(expectedOutput, s"Result did not match\n$queryString") { outputString } + } } + val broadcastHashJoinConf = Map(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760") + + val sortMergeJoinConf = Map( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.PREFER_SORTMERGEJOIN.key -> "true") + + val shuffleHashJoinConf = Map( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + "spark.sql.join.forceApplyShuffledHashJoin" -> "true") + + val shuffleConfSet: Set[Map[String, String]] = + Set(broadcastHashJoinConf, sortMergeJoinConf, shuffleHashJoinConf); + if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => val queryString = resourceToString(s"tpcds/$name.sql", classLoader = Thread.currentThread().getContextClassLoader) test(name) { val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") - runQuery(queryString, goldenFile) + runQuery(queryString, goldenFile, false) + if (!regenerateGoldenFiles) { + shuffleConfSet.foreach { testConf => + withSQLConf(testConf.toSeq: _*) { + logInfo(s"\n with configuration:\n${Utils.redact(testConf).mkString("\n")}") + runQuery(queryString, goldenFile, true) + } + } + } } } @@ -149,7 +178,14 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - runQuery(queryString, goldenFile) + runQuery(queryString, goldenFile, false) + if (!regenerateGoldenFiles) { + shuffleConfSet.foreach { testConf => + withSQLConf(testConf.toSeq: _*) { + runQuery(queryString, goldenFile, true) + } + } + } } } } else { From 0700055b768f85b27dc4b2b0b06c2815f52492e2 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Mon, 9 Aug 2021 23:20:22 +0800 Subject: [PATCH 02/12] modify --- .../test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index de0c6c73e731a..962be6905c695 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -136,7 +136,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val expectSorted = expectedOutput.split("\n").sorted.map(_.trim) .mkString("\n").replaceAll("\\s+$", "") val outputSorted = output.sorted.map(_.trim).mkString("\n").replaceAll("\\s+$", "") - assertResult(expectSorted, s"Result did not match") { outputSorted } + assertResult(expectSorted, s"Result did not match\n$queryString") { outputSorted } } else { assertResult(expectedOutput, s"Result did not match\n$queryString") { outputString } } From ec259530ff5bcaf603131b660aabbba9cbb15745 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Thu, 11 Nov 2021 00:00:26 +0800 Subject: [PATCH 03/12] fix --- .../scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 962be6905c695..4e46b8e6d229a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -152,7 +152,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", "spark.sql.join.forceApplyShuffledHashJoin" -> "true") - val shuffleConfSet: Set[Map[String, String]] = + val joinConfSet: Set[Map[String, String]] = Set(broadcastHashJoinConf, sortMergeJoinConf, shuffleHashJoinConf); if (tpcdsDataPath.nonEmpty) { @@ -163,7 +163,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") runQuery(queryString, goldenFile, false) if (!regenerateGoldenFiles) { - shuffleConfSet.foreach { testConf => + joinConfSet.foreach { testConf => withSQLConf(testConf.toSeq: _*) { logInfo(s"\n with configuration:\n${Utils.redact(testConf).mkString("\n")}") runQuery(queryString, goldenFile, true) @@ -180,7 +180,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") runQuery(queryString, goldenFile, false) if (!regenerateGoldenFiles) { - shuffleConfSet.foreach { testConf => + joinConfSet.foreach { testConf => withSQLConf(testConf.toSeq: _*) { runQuery(queryString, goldenFile, true) } From fc5f214b82530ca5c5b4064ebab4a29727f63ec9 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Thu, 11 Nov 2021 23:45:00 +0800 Subject: [PATCH 04/12] Change Default --- .../resources/tpcds-query-results/v1_4/q49.sql.out | 2 +- .../resources/tpcds-query-results/v1_4/q73.sql.out | 4 ++-- .../org/apache/spark/sql/TPCDSQueryTestSuite.scala | 10 +++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out index a1933e05565ca..e7e70dc11d4c1 100644 --- a/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out +++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q49.sql.out @@ -22,9 +22,9 @@ store 12783 0.88775510204081632653 6 6 store 11075 0.89743589743589743590 7 7 store 12889 0.95652173913043478261 8 8 store 1939 0.99000000000000000000 9 9 -store 12975 1.00000000000000000000 10 10 store 10455 1.00000000000000000000 10 10 store 4333 1.00000000000000000000 10 10 +store 12975 1.00000000000000000000 10 10 web 10485 0.48863636363636363636 1 1 web 4483 0.52688172043010752688 2 2 web 8833 0.58241758241758241758 3 3 diff --git a/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out b/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out index 615ac241c0e66..8294330ef9d9d 100644 --- a/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out +++ b/sql/core/src/test/resources/tpcds-query-results/v1_4/q73.sql.out @@ -3,7 +3,7 @@ -- !query schema struct -- !query output -Ransom Thomas Sir N 872 5 +Sauer Larry Mr. N 215795 5 Valle Chandra Dr. N 45338 5 Richardson Harry Mr. Y 85055 5 -Sauer Larry Mr. N 215795 5 +Ransom Thomas Sir N 872 5 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 4e46b8e6d229a..88b065f34f406 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -153,7 +153,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp "spark.sql.join.forceApplyShuffledHashJoin" -> "true") val joinConfSet: Set[Map[String, String]] = - Set(broadcastHashJoinConf, sortMergeJoinConf, shuffleHashJoinConf); + Set(broadcastHashJoinConf, shuffleHashJoinConf); if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => @@ -161,7 +161,9 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(name) { val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") - runQuery(queryString, goldenFile, false) + withSQLConf(sortMergeJoinConf.toSeq: _*) { + runQuery(queryString, goldenFile, false) + } if (!regenerateGoldenFiles) { joinConfSet.foreach { testConf => withSQLConf(testConf.toSeq: _*) { @@ -178,7 +180,9 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - runQuery(queryString, goldenFile, false) + withSQLConf(sortMergeJoinConf.toSeq: _*) { + runQuery(queryString, goldenFile, false) + } if (!regenerateGoldenFiles) { joinConfSet.foreach { testConf => withSQLConf(testConf.toSeq: _*) { From 31de361b330a1b41341c37deb0c101af8e4575da Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 20:36:21 +0800 Subject: [PATCH 05/12] Fix --- .../spark/sql/TPCDSQueryTestSuite.scala | 136 ++++++++++-------- 1 file changed, 77 insertions(+), 59 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 88b065f34f406..fa1b9d366f618 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -98,62 +98,83 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp """.stripMargin) } - private def runQuery(query: String, goldenFile: File, needSort: Boolean): Unit = { - val (schema, output) = handleExceptions(getNormalizedResult(spark, query)) - val queryString = query.trim - val outputString = output.mkString("\n").replaceAll("\\s+$", "") - if (regenerateGoldenFiles) { - val goldenOutput = { - s"-- Automatically generated by ${getClass.getSimpleName}\n\n" + - s"-- !query schema\n" + - schema + "\n" + - s"-- !query output\n" + - outputString + - "\n" - } - val parent = goldenFile.getParentFile - if (!parent.exists()) { - assert(parent.mkdirs(), "Could not create directory: " + parent) - } - stringToFile(goldenFile, goldenOutput) - } + private def runQuery(query: String, goldenFile: File, conf: Seq[(String, String)]): Unit = { + withSQLConf(conf: _*) { + try { + val (schema, output) = handleExceptions(getNormalizedResult(spark, query)) + val queryString = query.trim + val outputString = output.mkString("\n").replaceAll("\\s+$", "") + if (regenerateGoldenFiles) { + val goldenOutput = { + s"-- Automatically generated by ${getClass.getSimpleName}\n\n" + + s"-- !query schema\n" + + schema + "\n" + + s"-- !query output\n" + + outputString + + "\n" + } + val parent = goldenFile.getParentFile + if (!parent.exists()) { + assert(parent.mkdirs(), "Could not create directory: " + parent) + } + stringToFile(goldenFile, goldenOutput) + } - // Read back the golden file. - val (expectedSchema, expectedOutput) = { - val goldenOutput = fileToString(goldenFile) - val segments = goldenOutput.split("-- !query.*\n") + // Read back the golden file. + val (expectedSchema, expectedOutput) = { + val goldenOutput = fileToString(goldenFile) + val segments = goldenOutput.split("-- !query.*\n") - // query has 3 segments, plus the header - assert(segments.size == 3, - s"Expected 3 blocks in result file but got ${segments.size}. " + - "Try regenerate the result files.") + // query has 3 segments, plus the header + assert(segments.size == 3, + s"Expected 3 blocks in result file but got ${segments.size}. " + + "Try regenerate the result files.") - (segments(1).trim, segments(2).replaceAll("\\s+$", "")) - } + (segments(1).trim, segments(2).replaceAll("\\s+$", "")) + } - assertResult(expectedSchema, s"Schema did not match\n$queryString") { schema } - if (needSort) { - val expectSorted = expectedOutput.split("\n").sorted.map(_.trim) - .mkString("\n").replaceAll("\\s+$", "") - val outputSorted = output.sorted.map(_.trim).mkString("\n").replaceAll("\\s+$", "") - assertResult(expectSorted, s"Result did not match\n$queryString") { outputSorted } - } else { - assertResult(expectedOutput, s"Result did not match\n$queryString") { outputString } + assertResult(expectedSchema, s"Schema did not match\n$queryString") { + schema + } + if (needSort) { + val expectSorted = expectedOutput.split("\n").sorted.map(_.trim) + .mkString("\n").replaceAll("\\s+$", "") + val outputSorted = output.sorted.map(_.trim).mkString("\n").replaceAll("\\s+$", "") + assertResult(expectSorted, s"Result did not match\n$queryString") { + outputSorted + } + } else { + assertResult(expectedOutput, s"Result did not match\n$queryString") { + outputString + } + } + } catch { + case e: Throwable => + val configs = conf.map { + case (k, v) => s"$k=$v" + } + logError(s"Error using configs:\n${configs.mkString("\n")}") + throw e + } } } - val broadcastHashJoinConf = Map(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760") - val sortMergeJoinConf = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "true") + SQLConf.PREFER_SORTMERGEJOIN.key -> "true", + "spark.sql.needSortTestResults" -> "false") + + val broadcastHashJoinConf = Map( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760", + "spark.sql.needSortTestResults" -> "true") val shuffleHashJoinConf = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - "spark.sql.join.forceApplyShuffledHashJoin" -> "true") + "spark.sql.join.forceApplyShuffledHashJoin" -> "true", + "spark.sql.needSortTestResults" -> "true") val joinConfSet: Set[Map[String, String]] = - Set(broadcastHashJoinConf, shuffleHashJoinConf); + Set(sortMergeJoinConf, broadcastHashJoinConf, shuffleHashJoinConf); if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => @@ -161,15 +182,11 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(name) { val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") - withSQLConf(sortMergeJoinConf.toSeq: _*) { - runQuery(queryString, goldenFile, false) - } - if (!regenerateGoldenFiles) { - joinConfSet.foreach { testConf => - withSQLConf(testConf.toSeq: _*) { - logInfo(s"\n with configuration:\n${Utils.redact(testConf).mkString("\n")}") - runQuery(queryString, goldenFile, true) - } + if (regenerateGoldenFiles) { + runQuery(queryString, goldenFile, joinConfSet.head.toSeq) + } else { + joinConfSet.foreach { conf => + runQuery(queryString, goldenFile, conf.toSeq) } } } @@ -180,14 +197,11 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - withSQLConf(sortMergeJoinConf.toSeq: _*) { - runQuery(queryString, goldenFile, false) - } - if (!regenerateGoldenFiles) { - joinConfSet.foreach { testConf => - withSQLConf(testConf.toSeq: _*) { - runQuery(queryString, goldenFile, true) - } + if (regenerateGoldenFiles) { + runQuery(queryString, goldenFile, joinConfSet.head.toSeq) + } else { + joinConfSet.foreach { conf => + runQuery(queryString, goldenFile, conf.toSeq) } } } @@ -195,4 +209,8 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp } else { ignore("skipped because env `SPARK_TPCDS_DATA` is not set") {} } + + private def needSort(): Boolean = { + SQLConf.get.getConfString("spark.sql.needSortTestResults", "false").toBoolean + } } From 08c5fc3b484141492f50486ee9b5230f8fec3151 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 20:40:03 +0800 Subject: [PATCH 06/12] remove unused import --- .../test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index fa1b9d366f618..ce4a24211026f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -24,7 +24,6 @@ import org.apache.spark.{SparkConf, SparkContext} import org.apache.spark.sql.catalyst.util.{fileToString, resourceToString, stringToFile} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.TestSparkSession -import org.apache.spark.util.Utils /** * End-to-end tests to check TPCDS query results. From d2947c5358d963f0ea9a16aa2e9cb415e2115877 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 22:04:56 +0800 Subject: [PATCH 07/12] Fix --- .../spark/sql/TPCDSQueryTestSuite.scala | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index ce4a24211026f..23a48ff9e113f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -97,7 +97,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp """.stripMargin) } - private def runQuery(query: String, goldenFile: File, conf: Seq[(String, String)]): Unit = { + private def runQuery(query: String, goldenFile: File, conf: Seq[(String, String)], needSort: Boolean): Unit = { withSQLConf(conf: _*) { try { val (schema, output) = handleExceptions(getNormalizedResult(spark, query)) @@ -152,28 +152,23 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val configs = conf.map { case (k, v) => s"$k=$v" } - logError(s"Error using configs:\n${configs.mkString("\n")}") - throw e + throw new Exception(s"${e.getMessage} \nError using configs:\n${configs.mkString("\n")}") } } } val sortMergeJoinConf = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.PREFER_SORTMERGEJOIN.key -> "true", - "spark.sql.needSortTestResults" -> "false") + SQLConf.PREFER_SORTMERGEJOIN.key -> "true") - val broadcastHashJoinConf = Map( - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760", - "spark.sql.needSortTestResults" -> "true") + val broadcastHashJoinConf = Map(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760") val shuffleHashJoinConf = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - "spark.sql.join.forceApplyShuffledHashJoin" -> "true", - "spark.sql.needSortTestResults" -> "true") + "spark.sql.join.forceApplyShuffledHashJoin" -> "true") val joinConfSet: Set[Map[String, String]] = - Set(sortMergeJoinConf, broadcastHashJoinConf, shuffleHashJoinConf); + Set(broadcastHashJoinConf, shuffleHashJoinConf); if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => @@ -181,11 +176,10 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(name) { val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") - if (regenerateGoldenFiles) { - runQuery(queryString, goldenFile, joinConfSet.head.toSeq) - } else { + runQuery(queryString, goldenFile, sortMergeJoinConf.toSeq, false) + if (!regenerateGoldenFiles) { joinConfSet.foreach { conf => - runQuery(queryString, goldenFile, conf.toSeq) + runQuery(queryString, goldenFile, conf.toSeq, true) } } } @@ -196,11 +190,10 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - if (regenerateGoldenFiles) { - runQuery(queryString, goldenFile, joinConfSet.head.toSeq) - } else { + runQuery(queryString, goldenFile, joinConfSet.head.toSeq, false) + if (!regenerateGoldenFiles) { joinConfSet.foreach { conf => - runQuery(queryString, goldenFile, conf.toSeq) + runQuery(queryString, goldenFile, conf.toSeq, true) } } } @@ -208,8 +201,4 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp } else { ignore("skipped because env `SPARK_TPCDS_DATA` is not set") {} } - - private def needSort(): Boolean = { - SQLConf.get.getConfString("spark.sql.needSortTestResults", "false").toBoolean - } } From 5e4d6117df84466f5b8148cb67d17a179b592d12 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 22:07:27 +0800 Subject: [PATCH 08/12] Fix --- .../test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 23a48ff9e113f..280f5bbcc0861 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -190,7 +190,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - runQuery(queryString, goldenFile, joinConfSet.head.toSeq, false) + runQuery(queryString, goldenFile, sortMergeJoinConf.toSeq, false) if (!regenerateGoldenFiles) { joinConfSet.foreach { conf => runQuery(queryString, goldenFile, conf.toSeq, true) From 4da51b5a66d0e68ccd9c8ee1dfdb78dbae92abb5 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 22:26:24 +0800 Subject: [PATCH 09/12] Fix --- .../org/apache/spark/sql/TPCDSQueryTestSuite.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 280f5bbcc0861..c81359eac2519 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -152,7 +152,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val configs = conf.map { case (k, v) => s"$k=$v" } - throw new Exception(s"${e.getMessage} \nError using configs:\n${configs.mkString("\n")}") + throw new Exception(s"${e.getMessage}\nError using configs:\n${configs.mkString("\n")}") } } } @@ -168,7 +168,7 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp "spark.sql.join.forceApplyShuffledHashJoin" -> "true") val joinConfSet: Set[Map[String, String]] = - Set(broadcastHashJoinConf, shuffleHashJoinConf); + Set(sortMergeJoinConf, broadcastHashJoinConf, shuffleHashJoinConf); if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => @@ -176,9 +176,9 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(name) { val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out") - runQuery(queryString, goldenFile, sortMergeJoinConf.toSeq, false) + runQuery(queryString, goldenFile, joinConfSet.head.toSeq, false) if (!regenerateGoldenFiles) { - joinConfSet.foreach { conf => + joinConfSet.tail.foreach { conf => runQuery(queryString, goldenFile, conf.toSeq, true) } } @@ -190,9 +190,9 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp classLoader = Thread.currentThread().getContextClassLoader) test(s"$name-v2.7") { val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out") - runQuery(queryString, goldenFile, sortMergeJoinConf.toSeq, false) + runQuery(queryString, goldenFile, joinConfSet.head.toSeq, false) if (!regenerateGoldenFiles) { - joinConfSet.foreach { conf => + joinConfSet.tail.foreach { conf => runQuery(queryString, goldenFile, conf.toSeq, true) } } From 2f4325270f019b4c165c71e883c1315727f894dd Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Fri, 12 Nov 2021 22:32:30 +0800 Subject: [PATCH 10/12] Fix --- .../scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index c81359eac2519..922bebc0d3315 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -97,7 +97,11 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp """.stripMargin) } - private def runQuery(query: String, goldenFile: File, conf: Seq[(String, String)], needSort: Boolean): Unit = { + private def runQuery( + query: String, + goldenFile: File, + conf: Seq[(String, String)], + needSort: Boolean): Unit = { withSQLConf(conf: _*) { try { val (schema, output) = handleExceptions(getNormalizedResult(spark, query)) From c4ff4ab9d929a6b88af62ebe26659d756fe78cf4 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Sat, 13 Nov 2021 13:20:29 +0800 Subject: [PATCH 11/12] typo --- .../test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala index 922bebc0d3315..b6b34a2d95f1a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala @@ -167,12 +167,12 @@ class TPCDSQueryTestSuite extends QueryTest with TPCDSBase with SQLQueryTestHelp val broadcastHashJoinConf = Map(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "10485760") - val shuffleHashJoinConf = Map( + val shuffledHashJoinConf = Map( SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", "spark.sql.join.forceApplyShuffledHashJoin" -> "true") val joinConfSet: Set[Map[String, String]] = - Set(sortMergeJoinConf, broadcastHashJoinConf, shuffleHashJoinConf); + Set(sortMergeJoinConf, broadcastHashJoinConf, shuffledHashJoinConf); if (tpcdsDataPath.nonEmpty) { tpcdsQueries.foreach { name => From 510fa612cacaa9ec374392f56b4a74857e7632b8 Mon Sep 17 00:00:00 2001 From: RoryQi <1242949407@qq.com> Date: Mon, 15 Nov 2021 14:07:04 +0800 Subject: [PATCH 12/12] trigger test