Skip to content

[SPARK-24251][SQL] Add analysis tests for AppendData. #22043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-24251][SQL] Add analysis tests for AppendData. #22043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e58d4fc
SPARK-24251: Add analysis tests for AppendData.
rdblue Aug 8, 2018
765c5b4
SPARK-24251: Update for review comments.
rdblue Aug 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2258,8 +2258,8 @@ class Analyzer(
if (expected.size < query.output.size) {
throw new AnalysisException(
s"""Cannot write to '$tableName', too many data columns:
|Table columns: ${expected.map(_.name).mkString(", ")}
|Data columns: ${query.output.map(_.name).mkString(", ")}""".stripMargin)
|Table columns: ${expected.map(c => s"'${c.name}'").mkString(", ")}
|Data columns: ${query.output.map(c => s"'${c.name}'").mkString(", ")}""".stripMargin)
}

val errors = new mutable.ArrayBuffer[String]()
Expand All @@ -2278,8 +2278,9 @@ class Analyzer(
if (expected.size > query.output.size) {
throw new AnalysisException(
s"""Cannot write to '$tableName', not enough data columns:
|Table columns: ${expected.map(_.name).mkString(", ")}
|Data columns: ${query.output.map(_.name).mkString(", ")}""".stripMargin)
|Table columns: ${expected.map(c => s"'${c.name}'").mkString(", ")}
|Data columns: ${query.output.map(c => s"'${c.name}'").mkString(", ")}"""
.stripMargin)
}

query.output.zip(expected).flatMap {
Expand All @@ -2301,12 +2302,15 @@ class Analyzer(
queryExpr: NamedExpression,
addError: String => Unit): Option[NamedExpression] = {

// run the type check first to ensure type errors are present
val canWrite = DataType.canWrite(
queryExpr.dataType, tableAttr.dataType, resolver, tableAttr.name, addError)

if (queryExpr.nullable && !tableAttr.nullable) {
addError(s"Cannot write nullable values to non-null column '${tableAttr.name}'")
None

} else if (!DataType.canWrite(
tableAttr.dataType, queryExpr.dataType, resolver, tableAttr.name, addError)) {
} else if (!canWrite) {
None

} else {
Expand Down
15 changes: 8 additions & 7 deletions ...st/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,13 +363,14 @@ case class AppendData(
override def output: Seq[Attribute] = Seq.empty

override lazy val resolved: Boolean = {
query.output.size == table.output.size && query.output.zip(table.output).forall {
case (inAttr, outAttr) =>
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
DataType.equalsIgnoreCompatibleNullability(outAttr.dataType, inAttr.dataType) &&
(outAttr.nullable || !inAttr.nullable)
}
table.resolved && query.resolved && query.output.size == table.output.size &&
query.output.zip(table.output).forall {
case (inAttr, outAttr) =>
// names and types must match, nullability must be compatible
inAttr.name == outAttr.name &&
DataType.equalsIgnoreCompatibleNullability(outAttr.dataType, inAttr.dataType) &&
(outAttr.nullable || !inAttr.nullable)
}
}
}

Expand Down
Loading