Skip to content

Commit c2ba4af

Browse files
author
Michael Allman
committed
Clarify the definition of a partitioned table's file paths
1 parent 8a058c6 commit c2ba4af

File tree

1 file changed

+11
-1
lines changed

1 file changed

+11
-1
lines changed

sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveMetastoreCatalog.scala

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,17 @@ private[hive] class HiveMetastoreCatalog(sparkSession: SparkSession) extends Log
266266
}
267267
val partitionSpec = PartitionSpec(partitionSchema, partitions)
268268
val partitionPaths = partitions.map(_.path.toString)
269-
val paths = partitionPaths.padTo(1, metastoreRelation.hiveQlTable.getDataLocation.toString)
269+
270+
// By convention (for example, see MetaStorePartitionedTableFileCatalog), the definition of a
271+
// partitioned table's paths depends on whether that table has any actual partitions.
272+
// Partitioned tables without partitions use the location of the table's base path.
273+
// Partitioned tables with partitions use the locations of those partitions' data locations,
274+
// _omitting_ the table's base path.
275+
val paths = if (partitionPaths.isEmpty) {
276+
Seq(metastoreRelation.hiveQlTable.getDataLocation.toString)
277+
} else {
278+
partitionPaths
279+
}
270280

271281
val cached = getCached(
272282
tableIdentifier,

0 commit comments

Comments
 (0)