Skip to content

Commit 1bfaef8

Browse files
committed
fix
1 parent ce702c7 commit 1bfaef8

File tree

2 files changed

+7
-2
lines changed

2 files changed

+7
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/OptimizeMetadataOnlyQuery.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.spark.sql.execution
1919

20+
import java.util.Locale
21+
2022
import org.apache.spark.sql.AnalysisException
2123
import org.apache.spark.sql.catalyst.InternalRow
2224
import org.apache.spark.sql.catalyst.catalog.{HiveTableRelation, SessionCatalog}
@@ -81,9 +83,9 @@ case class OptimizeMetadataOnlyQuery(catalog: SessionCatalog) extends Rule[Logic
8183
private def getPartitionAttrs(
8284
partitionColumnNames: Seq[String],
8385
relation: LogicalPlan): Seq[Attribute] = {
84-
val attrMap = relation.output.map(_.name).zip(relation.output).toMap
86+
val attrMap = relation.output.map(_.name.toLowerCase(Locale.ROOT)).zip(relation.output).toMap
8587
partitionColumnNames.map { colName =>
86-
attrMap.getOrElse(colName,
88+
attrMap.getOrElse(colName.toLowerCase(Locale.ROOT),
8789
throw new AnalysisException(s"Unable to find the column `$colName` " +
8890
s"given [${relation.output.map(_.name).mkString(", ")}]")
8991
)

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/HadoopFsRelation.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ case class HadoopFsRelation(
6767
}
6868
}
6969

70+
// When data schema and partition schema have the overlapped columns, the output
71+
// schema respects the order of data schema for the overlapped columns, but respect
72+
// the data types of partition schema
7073
val schema: StructType = {
7174
StructType(dataSchema.map(f => overlappedPartCols.getOrElse(getColName(f), f)) ++
7275
partitionSchema.filterNot(f => overlappedPartCols.contains(getColName(f))))

0 commit comments

Comments
 (0)