Skip to content

Commit 6437d4b

Browse files
committed
Assembles requested schema from Parquet file schema
1 parent bcac49f commit 6437d4b

File tree

1 file changed

+8
-5
lines changed

1 file changed

+8
-5
lines changed

sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,17 +90,20 @@ private[parquet] class RowReadSupport extends ReadSupport[InternalRow] with Logg
9090
val maybeRowSchema = Option(conf.get(RowWriteSupport.SPARK_ROW_SCHEMA))
9191
val maybeRequestedSchema = Option(conf.get(RowReadSupport.SPARK_ROW_REQUESTED_SCHEMA))
9292

93-
val parquetSchema = maybeRequestedSchema.map { schemaString =>
94-
val toParquet = new CatalystSchemaConverter(conf)
95-
toParquet.convert(StructType.fromString(schemaString))
96-
}.getOrElse(context.getFileSchema)
93+
val parquetRequestedSchema =
94+
maybeRequestedSchema.map { schemaString =>
95+
StructType.fromString(schemaString).map { field =>
96+
val fieldType = context.getFileSchema.asGroupType().getType(field.name)
97+
new MessageType("root", fieldType)
98+
}.reduce(_ union _)
99+
}.getOrElse(context.getFileSchema)
97100

98101
val metadata =
99102
Map.empty[String, String] ++
100103
maybeRequestedSchema.map(RowReadSupport.SPARK_ROW_REQUESTED_SCHEMA -> _) ++
101104
maybeRowSchema.map(RowWriteSupport.SPARK_ROW_SCHEMA -> _)
102105

103-
new ReadContext(parquetSchema, metadata)
106+
new ReadContext(parquetRequestedSchema, metadata)
104107
}
105108
}
106109

0 commit comments

Comments
 (0)