File tree Expand file tree Collapse file tree 1 file changed +8
-5
lines changed
sql/core/src/main/scala/org/apache/spark/sql/parquet Expand file tree Collapse file tree 1 file changed +8
-5
lines changed Original file line number Diff line number Diff line change @@ -90,17 +90,20 @@ private[parquet] class RowReadSupport extends ReadSupport[InternalRow] with Logg
9090 val maybeRowSchema = Option (conf.get(RowWriteSupport .SPARK_ROW_SCHEMA ))
9191 val maybeRequestedSchema = Option (conf.get(RowReadSupport .SPARK_ROW_REQUESTED_SCHEMA ))
9292
93- val parquetSchema = maybeRequestedSchema.map { schemaString =>
94- val toParquet = new CatalystSchemaConverter (conf)
95- toParquet.convert(StructType .fromString(schemaString))
96- }.getOrElse(context.getFileSchema)
93+ val parquetRequestedSchema =
94+ maybeRequestedSchema.map { schemaString =>
95+ StructType .fromString(schemaString).map { field =>
96+ val fieldType = context.getFileSchema.asGroupType().getType(field.name)
97+ new MessageType (" root" , fieldType)
98+ }.reduce(_ union _)
99+ }.getOrElse(context.getFileSchema)
97100
98101 val metadata =
99102 Map .empty[String , String ] ++
100103 maybeRequestedSchema.map(RowReadSupport .SPARK_ROW_REQUESTED_SCHEMA -> _) ++
101104 maybeRowSchema.map(RowWriteSupport .SPARK_ROW_SCHEMA -> _)
102105
103- new ReadContext (parquetSchema , metadata)
106+ new ReadContext (parquetRequestedSchema , metadata)
104107 }
105108}
106109
You can’t perform that action at this time.
0 commit comments