Skip to content

Commit c25aa8f

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-16329][SQL][BACKPORT-1.6] Star Expansion over Table Containing No Column #14040
#### What changes were proposed in this pull request? Star expansion over a table containing zero column does not work since 1.6. However, it works in Spark 1.5.1. This PR is to fix the issue in the master branch. For example, ```scala val rddNoCols = sqlContext.sparkContext.parallelize(1 to 10).map(_ => Row.empty) val dfNoCols = sqlContext.createDataFrame(rddNoCols, StructType(Seq.empty)) dfNoCols.registerTempTable("temp_table_no_cols") sqlContext.sql("select * from temp_table_no_cols").show ``` Without the fix, users will get the following the exception: ``` java.lang.IllegalArgumentException: requirement failed at scala.Predef$.require(Predef.scala:221) at org.apache.spark.sql.catalyst.analysis.UnresolvedStar.expand(unresolved.scala:199) ``` #### How was this patch tested? Tests are added Author: gatorsmile <[email protected]> Closes #14042 from gatorsmile/starExpansionEmpty.
1 parent 1026aba commit c25aa8f

File tree

2 files changed

+37
-9
lines changed

2 files changed

+37
-9
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -180,23 +180,20 @@ abstract class Star extends LeafExpression with NamedExpression {
180180
case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevaluable {
181181

182182
override def expand(input: LogicalPlan, resolver: Resolver): Seq[NamedExpression] = {
183+
// If there is no table specified, use all input attributes.
184+
if (target.isEmpty) return input.output
183185

184-
// First try to expand assuming it is table.*.
185-
val expandedAttributes: Seq[Attribute] = target match {
186-
// If there is no table specified, use all input attributes.
187-
case None => input.output
188-
// If there is a table, pick out attributes that are part of this table.
189-
case Some(t) => if (t.size == 1) {
190-
input.output.filter(_.qualifiers.exists(resolver(_, t.head)))
186+
val expandedAttributes =
187+
if (target.get.size == 1) {
188+
// If there is a table, pick out attributes that are part of this table.
189+
input.output.filter(_.qualifiers.exists(resolver(_, target.get.head)))
191190
} else {
192191
List()
193192
}
194-
}
195193
if (expandedAttributes.nonEmpty) return expandedAttributes
196194

197195
// Try to resolve it as a struct expansion. If there is a conflict and both are possible,
198196
// (i.e. [name].* is both a table and a struct), the struct path can always be qualified.
199-
require(target.isDefined)
200197
val attribute = input.resolve(target.get, resolver)
201198
if (attribute.isDefined) {
202199
// This target resolved to an attribute in child. It must be a struct. Expand it.

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1950,6 +1950,37 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
19501950
}
19511951
}
19521952

1953+
test("Star Expansion - table with zero column") {
1954+
withTempTable("temp_table_no_cols") {
1955+
val rddNoCols = sparkContext.parallelize(1 to 10).map(_ => Row.empty)
1956+
val dfNoCols = sqlContext.createDataFrame(rddNoCols, StructType(Seq.empty))
1957+
dfNoCols.registerTempTable("temp_table_no_cols")
1958+
1959+
// ResolvedStar
1960+
checkAnswer(
1961+
dfNoCols,
1962+
dfNoCols.select(dfNoCols.col("*")))
1963+
1964+
// UnresolvedStar
1965+
checkAnswer(
1966+
dfNoCols,
1967+
sql("SELECT * FROM temp_table_no_cols"))
1968+
checkAnswer(
1969+
dfNoCols,
1970+
dfNoCols.select($"*"))
1971+
1972+
var e = intercept[AnalysisException] {
1973+
sql("SELECT a.* FROM temp_table_no_cols a")
1974+
}.getMessage
1975+
assert(e.contains("cannot resolve 'a.*' give input columns ''"))
1976+
1977+
e = intercept[AnalysisException] {
1978+
dfNoCols.select($"b.*")
1979+
}.getMessage
1980+
assert(e.contains("cannot resolve 'b.*' give input columns ''"))
1981+
}
1982+
}
1983+
19531984
test("Common subexpression elimination") {
19541985
// select from a table to prevent constant folding.
19551986
val df = sql("SELECT a, b from testData2 limit 1")

0 commit comments

Comments
 (0)