Skip to content

Commit 28d130d

Browse files
committed
Follow up of SPARK-16034.
1 parent 1e43dff commit 28d130d

File tree

2 files changed

+23
-6
lines changed

2 files changed

+23
-6
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSource.scala

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,7 @@ case class DataSource(
435435
// If we are appending to a table that already exists, make sure the partitioning matches
436436
// up. If we fail to load the table for whatever reason, ignore the check.
437437
if (mode == SaveMode.Append) {
438-
val existingColumns = Try {
438+
val existingPartitionColumns = Try {
439439
resolveRelation()
440440
.asInstanceOf[HadoopFsRelation]
441441
.location
@@ -444,13 +444,14 @@ case class DataSource(
444444
.fieldNames
445445
.toSeq
446446
}.getOrElse(Seq.empty[String])
447+
// TODO: Case sensitivity.
447448
val sameColumns =
448-
existingColumns.map(_.toLowerCase) == partitionColumns.map(_.toLowerCase)
449-
if (existingColumns.size > 0 && !sameColumns) {
449+
existingPartitionColumns.map(_.toLowerCase) == partitionColumns.map(_.toLowerCase)
450+
if (existingPartitionColumns.size > 0 && !sameColumns) {
450451
throw new AnalysisException(
451452
s"""Requested partitioning does not match existing partitioning.
452453
|Existing partitioning columns:
453-
| ${existingColumns.mkString(", ")}
454+
| ${existingPartitionColumns.mkString(", ")}
454455
|Requested partitioning columns:
455456
| ${partitionColumns.mkString(", ")}
456457
|""".stripMargin)

sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1337,8 +1337,24 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach {
13371337
assert(sql("select * from partitionedTable").collect().size == 1)
13381338
// Inserts new data successfully when partition columns are correctly specified in
13391339
// partitionBy(...).
1340-
df.write.mode("append").partitionBy("a", "b").saveAsTable("partitionedTable")
1341-
assert(sql("select * from partitionedTable").collect().size == 2)
1340+
// TODO: Right now, partition columns are always treated in a case-insensitive way.
1341+
// See the write method in DataSource.scala.
1342+
Seq((4, 5, 6)).toDF("a", "B", "c")
1343+
.write
1344+
.mode("append")
1345+
.partitionBy("a", "B")
1346+
.saveAsTable("partitionedTable")
1347+
1348+
Seq((7, 8, 9)).toDF("a", "b", "c")
1349+
.write
1350+
.mode("append")
1351+
.partitionBy("a", "b")
1352+
.saveAsTable("partitionedTable")
1353+
1354+
checkAnswer(
1355+
sql("select a, b, c from partitionedTable"),
1356+
Row(1, 2, 3) :: Row(4, 5, 6) :: Row(7, 8, 9) :: Nil
1357+
)
13421358
}
13431359
}
13441360
}

0 commit comments

Comments
 (0)