Skip to content

Commit d5683a7

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-16329][SQL][BACKPORT-2.0] Star Expansion over Table Containing No Column
#### What changes were proposed in this pull request? Star expansion over a table containing zero column does not work since 1.6. However, it works in Spark 1.5.1. This PR is to fix the issue in the master branch. For example, ```scala val rddNoCols = sqlContext.sparkContext.parallelize(1 to 10).map(_ => Row.empty) val dfNoCols = sqlContext.createDataFrame(rddNoCols, StructType(Seq.empty)) dfNoCols.registerTempTable("temp_table_no_cols") sqlContext.sql("select * from temp_table_no_cols").show ``` Without the fix, users will get the following the exception: ``` java.lang.IllegalArgumentException: requirement failed at scala.Predef$.require(Predef.scala:221) at org.apache.spark.sql.catalyst.analysis.UnresolvedStar.expand(unresolved.scala:199) ``` #### How was this patch tested? Tests are added Author: gatorsmile <[email protected]> Closes #14040 from gatorsmile/starExpansionEmptyTable.
1 parent ecbb447 commit d5683a7

File tree

2 files changed

+37
-9
lines changed

2 files changed

+37
-9
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -215,23 +215,20 @@ abstract class Star extends LeafExpression with NamedExpression {
215215
case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevaluable {
216216

217217
override def expand(input: LogicalPlan, resolver: Resolver): Seq[NamedExpression] = {
218+
// If there is no table specified, use all input attributes.
219+
if (target.isEmpty) return input.output
218220

219-
// First try to expand assuming it is table.*.
220-
val expandedAttributes: Seq[Attribute] = target match {
221-
// If there is no table specified, use all input attributes.
222-
case None => input.output
223-
// If there is a table, pick out attributes that are part of this table.
224-
case Some(t) => if (t.size == 1) {
225-
input.output.filter(_.qualifier.exists(resolver(_, t.head)))
221+
val expandedAttributes =
222+
if (target.get.size == 1) {
223+
// If there is a table, pick out attributes that are part of this table.
224+
input.output.filter(_.qualifier.exists(resolver(_, target.get.head)))
226225
} else {
227226
List()
228227
}
229-
}
230228
if (expandedAttributes.nonEmpty) return expandedAttributes
231229

232230
// Try to resolve it as a struct expansion. If there is a conflict and both are possible,
233231
// (i.e. [name].* is both a table and a struct), the struct path can always be qualified.
234-
require(target.isDefined)
235232
val attribute = input.resolve(target.get, resolver)
236233
if (attribute.isDefined) {
237234
// This target resolved to an attribute in child. It must be a struct. Expand it.

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2115,6 +2115,37 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
21152115
}
21162116
}
21172117

2118+
test("Star Expansion - table with zero column") {
2119+
withTempTable("temp_table_no_cols") {
2120+
val rddNoCols = sparkContext.parallelize(1 to 10).map(_ => Row.empty)
2121+
val dfNoCols = spark.createDataFrame(rddNoCols, StructType(Seq.empty))
2122+
dfNoCols.createTempView("temp_table_no_cols")
2123+
2124+
// ResolvedStar
2125+
checkAnswer(
2126+
dfNoCols,
2127+
dfNoCols.select(dfNoCols.col("*")))
2128+
2129+
// UnresolvedStar
2130+
checkAnswer(
2131+
dfNoCols,
2132+
sql("SELECT * FROM temp_table_no_cols"))
2133+
checkAnswer(
2134+
dfNoCols,
2135+
dfNoCols.select($"*"))
2136+
2137+
var e = intercept[AnalysisException] {
2138+
sql("SELECT a.* FROM temp_table_no_cols a")
2139+
}.getMessage
2140+
assert(e.contains("cannot resolve 'a.*' give input columns ''"))
2141+
2142+
e = intercept[AnalysisException] {
2143+
dfNoCols.select($"b.*")
2144+
}.getMessage
2145+
assert(e.contains("cannot resolve 'b.*' give input columns ''"))
2146+
}
2147+
}
2148+
21182149
test("Common subexpression elimination") {
21192150
// TODO: support subexpression elimination in whole stage codegen
21202151
withSQLConf("spark.sql.codegen.wholeStage" -> "false") {

0 commit comments

Comments
 (0)