From a46144a4e3d10c4b2d872250518fe06f659eb90f Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Wed, 17 Jun 2015 11:14:33 -0700 Subject: [PATCH 1/7] WIP --- .../spark/sql/execution/SortSuite.scala | 45 ++++++ .../spark/sql/execution/SparkPlanTest.scala | 140 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala new file mode 100644 index 0000000000000..bd42917162bdd --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution + +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.{Ascending, SortOrder} + +class SortSuite extends SparkPlanTest { + + test("basic sorting using ExternalSort") { + + val input = Seq( + ("Hello", 4), + ("Hello", 1), + ("World", 8) + ) + + val sortOrder = Seq( + SortOrder('_1, Ascending), + SortOrder('_2, Ascending) + ) + + checkAnswer( + input, + child => new ExternalSort(sortOrder, global = false, child), + input.sorted + ) + + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala new file mode 100644 index 0000000000000..93165b17f5550 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution + +import scala.util.control.NonFatal +import scala.reflect.runtime.universe.TypeTag + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.test.TestSQLContext +import org.apache.spark.sql.{Row, DataFrame} +import org.apache.spark.sql.catalyst.util._ + +/** + * Base class for writing tests for individual physical operators. For an example of how this class + * can be used, see [[SortSuite]]. + */ +class SparkPlanTest extends SparkFunSuite { + + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param input the input data to be used. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the + * physical operator that's being tested. + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + protected def checkAnswer( + input: DataFrame, + planFunction: SparkPlan => SparkPlan, + expectedAnswer: Seq[Row]): Unit = { + SparkPlanTest.checkAnswer(input, planFunction, expectedAnswer) match { + case Some(errorMessage) => fail(errorMessage) + case None => + } + } + + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param input the input data to be used. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the + * physical operator that's being tested. + * @param expectedAnswer the expected result in a [[Seq]] of [[Product]]s. + */ + protected def checkAnswer[A <: Product : TypeTag]( + input: Seq[A], + planFunction: SparkPlan => SparkPlan, + expectedAnswer: Seq[A]): Unit = { + val inputDf = TestSQLContext.createDataFrame(input) + val expectedRows = expectedAnswer.map(t => Row.apply(t)) + SparkPlanTest.checkAnswer(inputDf, planFunction, expectedRows) match { + case Some(errorMessage) => fail(errorMessage) + case None => + } + } +} + +/** + * Helper methods for writing tests of individual physical operators. + */ +object SparkPlanTest { + + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param input the input data to be used. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the + * physical operator that's being tested. + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + def checkAnswer( + input: DataFrame, + planFunction: SparkPlan => SparkPlan, + expectedAnswer: Seq[Row]): Option[String] = { + + val outputPlan = planFunction(input.queryExecution.sparkPlan) + + def prepareAnswer(answer: Seq[Row]): Seq[Row] = { + // Converts data to types that we can do equality comparison using Scala collections. + // For BigDecimal type, the Scala type has a better definition of equality test (similar to + // Java's java.math.BigDecimal.compareTo). + // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for + // equality test. + // This function is copied from Catalyst's QueryTest + val converted: Seq[Row] = answer.map { s => + Row.fromSeq(s.toSeq.map { + case d: java.math.BigDecimal => BigDecimal(d) + case b: Array[Byte] => b.toSeq + case o => o + }) + } + converted.sortBy(_.toString()) + } + + val sparkAnswer: Seq[Row] = try { + outputPlan.executeCollect().toSeq + } catch { + case NonFatal(e) => + val errorMessage = + s""" + | Exception thrown while executing Spark plan: + | $outputPlan + | == Exception == + | $e + | ${org.apache.spark.sql.catalyst.util.stackTraceToString(e)} + """.stripMargin + return Some(errorMessage) + } + + if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) { + val errorMessage = + s""" + | Results do not match for Spark plan: + | $outputPlan + | == Results == + | ${sideBySide( + s"== Correct Answer - ${expectedAnswer.size} ==" +: + prepareAnswer(expectedAnswer).map(_.toString()), + s"== Spark Answer - ${sparkAnswer.size} ==" +: + prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n")} + """.stripMargin + return Some(errorMessage) + } + + None + } +} + From 996332ab0ff6432110f134efaf622d2fa10561b8 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 18 Jun 2015 10:36:34 -0700 Subject: [PATCH 2/7] Add types so that tests compile --- .../apache/spark/sql/execution/SortSuite.scala | 2 +- .../spark/sql/execution/SparkPlanTest.scala | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index bd42917162bdd..a890904f90a85 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -37,7 +37,7 @@ class SortSuite extends SparkPlanTest { checkAnswer( input, - child => new ExternalSort(sortOrder, global = false, child), + (child: SparkPlan) => new ExternalSort(sortOrder, global = false, child), input.sorted ) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala index 93165b17f5550..ed8c761b69093 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -26,16 +26,16 @@ import org.apache.spark.sql.{Row, DataFrame} import org.apache.spark.sql.catalyst.util._ /** - * Base class for writing tests for individual physical operators. For an example of how this class - * can be used, see [[SortSuite]]. + * Base class for writing tests for individual physical operators. For an example of how this + * class's test helper methods can be used, see [[SortSuite]]. */ class SparkPlanTest extends SparkFunSuite { /** * Runs the plan and makes sure the answer matches the expected result. * @param input the input data to be used. - * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the - * physical operator that's being tested. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate + * the physical operator that's being tested. * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. */ protected def checkAnswer( @@ -51,8 +51,8 @@ class SparkPlanTest extends SparkFunSuite { /** * Runs the plan and makes sure the answer matches the expected result. * @param input the input data to be used. - * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the - * physical operator that's being tested. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate + * the physical operator that's being tested. * @param expectedAnswer the expected result in a [[Seq]] of [[Product]]s. */ protected def checkAnswer[A <: Product : TypeTag]( @@ -76,8 +76,8 @@ object SparkPlanTest { /** * Runs the plan and makes sure the answer matches the expected result. * @param input the input data to be used. - * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate the - * physical operator that's being tested. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate + * the physical operator that's being tested. * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. */ def checkAnswer( From c60a44d4c9b59a613764484dedbe1cbbc514ad9b Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 18 Jun 2015 11:01:16 -0700 Subject: [PATCH 3/7] Manually bind references --- .../scala/org/apache/spark/sql/execution/SortSuite.scala | 8 ++++---- .../org/apache/spark/sql/execution/SparkPlanTest.scala | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index a890904f90a85..5f2db13d8202d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -17,8 +17,8 @@ package org.apache.spark.sql.execution -import org.apache.spark.sql.catalyst.dsl.expressions._ -import org.apache.spark.sql.catalyst.expressions.{Ascending, SortOrder} +import org.apache.spark.sql.catalyst.expressions.{BoundReference, Ascending, SortOrder} +import org.apache.spark.sql.types.{IntegerType, StringType} class SortSuite extends SparkPlanTest { @@ -31,8 +31,8 @@ class SortSuite extends SparkPlanTest { ) val sortOrder = Seq( - SortOrder('_1, Ascending), - SortOrder('_2, Ascending) + SortOrder(BoundReference(0, StringType, nullable = false), Ascending), + SortOrder(BoundReference(1, IntegerType, nullable = false), Ascending) ) checkAnswer( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala index ed8c761b69093..b4f37cf8f69ae 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -60,7 +60,7 @@ class SparkPlanTest extends SparkFunSuite { planFunction: SparkPlan => SparkPlan, expectedAnswer: Seq[A]): Unit = { val inputDf = TestSQLContext.createDataFrame(input) - val expectedRows = expectedAnswer.map(t => Row.apply(t)) + val expectedRows = expectedAnswer.map(Row.fromTuple) SparkPlanTest.checkAnswer(inputDf, planFunction, expectedRows) match { case Some(errorMessage) => fail(errorMessage) case None => From d9ab1e46c457a378b623b2e249ef1b53e0289eae Mon Sep 17 00:00:00 2001 From: Michael Armbrust Date: Thu, 18 Jun 2015 20:49:58 +0000 Subject: [PATCH 4/7] Add simple resolver --- .../spark/sql/execution/SortSuite.scala | 20 +++++---- .../spark/sql/execution/SparkPlanTest.scala | 43 ++++++++++++++++++- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index 5f2db13d8202d..05be27437ae96 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -18,9 +18,13 @@ package org.apache.spark.sql.execution import org.apache.spark.sql.catalyst.expressions.{BoundReference, Ascending, SortOrder} +import org.apache.spark.sql.catalyst.dsl.expressions._ + +import org.apache.spark.sql.test.TestSQLContext import org.apache.spark.sql.types.{IntegerType, StringType} class SortSuite extends SparkPlanTest { + import TestSQLContext.implicits.localSeqToDataFrameHolder test("basic sorting using ExternalSort") { @@ -30,16 +34,14 @@ class SortSuite extends SparkPlanTest { ("World", 8) ) - val sortOrder = Seq( - SortOrder(BoundReference(0, StringType, nullable = false), Ascending), - SortOrder(BoundReference(1, IntegerType, nullable = false), Ascending) - ) - checkAnswer( - input, - (child: SparkPlan) => new ExternalSort(sortOrder, global = false, child), - input.sorted - ) + input.toDF("a", "b"), + ExternalSort('a.asc :: 'b.asc :: Nil, global = false, _: SparkPlan), + input.sorted) + checkAnswer( + input.toDF("a", "b"), + ExternalSort('b.asc :: 'a.asc :: Nil, global = false, _: SparkPlan), + input.sortBy(t => (t._2, t._1))) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala index b4f37cf8f69ae..39e561ec3e434 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -21,9 +21,13 @@ import scala.util.control.NonFatal import scala.reflect.runtime.universe.TypeTag import org.apache.spark.SparkFunSuite + +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.BoundReference +import org.apache.spark.sql.catalyst.util._ + import org.apache.spark.sql.test.TestSQLContext import org.apache.spark.sql.{Row, DataFrame} -import org.apache.spark.sql.catalyst.util._ /** * Base class for writing tests for individual physical operators. For an example of how this @@ -48,6 +52,24 @@ class SparkPlanTest extends SparkFunSuite { } } + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param input the input data to be used. + * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate + * the physical operator that's being tested. + * @param expectedAnswer the expected result in a [[Seq]] of [[Product]]s. + */ + protected def checkAnswer[A <: Product : TypeTag]( + input: DataFrame, + planFunction: SparkPlan => SparkPlan, + expectedAnswer: Seq[A]): Unit = { + val expectedRows = expectedAnswer.map(Row.fromTuple) + SparkPlanTest.checkAnswer(input, planFunction, expectedRows) match { + case Some(errorMessage) => fail(errorMessage) + case None => + } + } + /** * Runs the plan and makes sure the answer matches the expected result. * @param input the input data to be used. @@ -87,6 +109,23 @@ object SparkPlanTest { val outputPlan = planFunction(input.queryExecution.sparkPlan) + // A very simple resolver to make writing tests easier. In contrast to the real resolver + // this is always case sensitive and does not try to handle scoping or complex type resolution. + val resolvedPlan = outputPlan transform { + case plan: SparkPlan => + val inputMap = plan.children.flatMap(_.output).zipWithIndex.map { + case (a, i) => + (a.name, BoundReference(i, a.dataType, a.nullable)) + }.toMap + + plan.transformExpressions { + case UnresolvedAttribute(Seq(u)) => + inputMap.get(u).getOrElse { + sys.error(s"Invalid Test: Cannot resolve $u given input ${inputMap}") + } + } + } + def prepareAnswer(answer: Seq[Row]): Seq[Row] = { // Converts data to types that we can do equality comparison using Scala collections. // For BigDecimal type, the Scala type has a better definition of equality test (similar to @@ -105,7 +144,7 @@ object SparkPlanTest { } val sparkAnswer: Seq[Row] = try { - outputPlan.executeCollect().toSeq + resolvedPlan.executeCollect().toSeq } catch { case NonFatal(e) => val errorMessage = From ae1896bdeea9504494b539348ae2467d2906e900 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 18 Jun 2015 14:52:51 -0700 Subject: [PATCH 5/7] Provide implicits automatically --- .../org/apache/spark/sql/execution/SortSuite.scala | 5 ----- .../apache/spark/sql/execution/SparkPlanTest.scala | 12 ++++++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index 05be27437ae96..30cf4eb413907 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -17,14 +17,9 @@ package org.apache.spark.sql.execution -import org.apache.spark.sql.catalyst.expressions.{BoundReference, Ascending, SortOrder} import org.apache.spark.sql.catalyst.dsl.expressions._ -import org.apache.spark.sql.test.TestSQLContext -import org.apache.spark.sql.types.{IntegerType, StringType} - class SortSuite extends SparkPlanTest { - import TestSQLContext.implicits.localSeqToDataFrameHolder test("basic sorting using ExternalSort") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala index 39e561ec3e434..f2ba4065cd8c3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -17,8 +17,9 @@ package org.apache.spark.sql.execution -import scala.util.control.NonFatal +import scala.language.implicitConversions import scala.reflect.runtime.universe.TypeTag +import scala.util.control.NonFatal import org.apache.spark.SparkFunSuite @@ -27,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.BoundReference import org.apache.spark.sql.catalyst.util._ import org.apache.spark.sql.test.TestSQLContext -import org.apache.spark.sql.{Row, DataFrame} +import org.apache.spark.sql.{DataFrameHolder, Row, DataFrame} /** * Base class for writing tests for individual physical operators. For an example of how this @@ -35,6 +36,13 @@ import org.apache.spark.sql.{Row, DataFrame} */ class SparkPlanTest extends SparkFunSuite { + /** + * Creates a DataFrame from a local Seq of Product. + */ + implicit def localSeqToDataFrameHolder[A <: Product : TypeTag](data: Seq[A]): DataFrameHolder = { + TestSQLContext.implicits.localSeqToDataFrameHolder(data) + } + /** * Runs the plan and makes sure the answer matches the expected result. * @param input the input data to be used. From 84214be7dc4b571cdc42d02deaebfa1a00fc0a00 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 18 Jun 2015 14:53:54 -0700 Subject: [PATCH 6/7] Add an extra column which isn't part of the sort --- .../org/apache/spark/sql/execution/SortSuite.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index 30cf4eb413907..054712c457938 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -24,18 +24,18 @@ class SortSuite extends SparkPlanTest { test("basic sorting using ExternalSort") { val input = Seq( - ("Hello", 4), - ("Hello", 1), - ("World", 8) + ("Hello", 4, 2.0), + ("Hello", 1, 1.0), + ("World", 8, 3.0) ) checkAnswer( - input.toDF("a", "b"), + input.toDF("a", "b", "c"), ExternalSort('a.asc :: 'b.asc :: Nil, global = false, _: SparkPlan), input.sorted) checkAnswer( - input.toDF("a", "b"), + input.toDF("a", "b", "c"), ExternalSort('b.asc :: 'a.asc :: Nil, global = false, _: SparkPlan), input.sortBy(t => (t._2, t._1))) } From f8ce275d2795f61458e6237e2d7a6101fbfdcb7b Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Thu, 18 Jun 2015 14:57:12 -0700 Subject: [PATCH 7/7] Fix some IntelliJ inspections and delete some dead code --- .../spark/sql/execution/SortSuite.scala | 2 ++ .../spark/sql/execution/SparkPlanTest.scala | 24 ++----------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala index 054712c457938..a1e3ca11b1ad9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SortSuite.scala @@ -21,6 +21,8 @@ import org.apache.spark.sql.catalyst.dsl.expressions._ class SortSuite extends SparkPlanTest { + // This test was originally added as an example of how to use [[SparkPlanTest]]; + // it's not designed to be a comprehensive test of ExternalSort. test("basic sorting using ExternalSort") { val input = Seq( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala index f2ba4065cd8c3..13f3be8ca28d6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanTest.scala @@ -77,25 +77,6 @@ class SparkPlanTest extends SparkFunSuite { case None => } } - - /** - * Runs the plan and makes sure the answer matches the expected result. - * @param input the input data to be used. - * @param planFunction a function which accepts the input SparkPlan and uses it to instantiate - * the physical operator that's being tested. - * @param expectedAnswer the expected result in a [[Seq]] of [[Product]]s. - */ - protected def checkAnswer[A <: Product : TypeTag]( - input: Seq[A], - planFunction: SparkPlan => SparkPlan, - expectedAnswer: Seq[A]): Unit = { - val inputDf = TestSQLContext.createDataFrame(input) - val expectedRows = expectedAnswer.map(Row.fromTuple) - SparkPlanTest.checkAnswer(inputDf, planFunction, expectedRows) match { - case Some(errorMessage) => fail(errorMessage) - case None => - } - } } /** @@ -128,9 +109,8 @@ object SparkPlanTest { plan.transformExpressions { case UnresolvedAttribute(Seq(u)) => - inputMap.get(u).getOrElse { - sys.error(s"Invalid Test: Cannot resolve $u given input ${inputMap}") - } + inputMap.getOrElse(u, + sys.error(s"Invalid Test: Cannot resolve $u given input $inputMap")) } }