Skip to content

[SPARK-10136] [SQL] A more robust fix for SPARK-10136 #8361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

[SPARK-10136] [SQL] A more robust fix for SPARK-10136 #8361

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
15cd105
Fixes SPARK-10136 in a more proper way
liancheng Aug 21, 2015
74802cd
Minor refactoring for readability
liancheng Aug 22, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions .../main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,8 +415,9 @@ private[parquet] class CatalystRowConverter(
private val elementConverter: Converter = {
val repeatedType = parquetSchema.getType(0)
val elementType = catalystSchema.elementType
val parentName = parquetSchema.getName

if (isElementType(repeatedType, elementType)) {
if (isElementType(repeatedType, elementType, parentName)) {
newConverter(repeatedType, elementType, new ParentContainerUpdater {
override def set(value: Any): Unit = currentArray += value
})
Expand Down Expand Up @@ -453,10 +454,13 @@ private[parquet] class CatalystRowConverter(
* @see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
*/
// scalastyle:on
private def isElementType(parquetRepeatedType: Type, catalystElementType: DataType): Boolean = {
private def isElementType(
parquetRepeatedType: Type, catalystElementType: DataType, parentName: String): Boolean = {
(parquetRepeatedType, catalystElementType) match {
case (t: PrimitiveType, _) => true
case (t: GroupType, _) if t.getFieldCount > 1 => true
case (t: GroupType, _) if t.getFieldCount == 1 && t.getName == "array" => true
case (t: GroupType, _) if t.getFieldCount == 1 && t.getName == parentName + "_tuple" => true
case (t: GroupType, StructType(Array(f))) if f.name == t.getFieldName(0) => true
case _ => false
}
Expand All @@ -474,15 +478,9 @@ private[parquet] class CatalystRowConverter(

override def getConverter(fieldIndex: Int): Converter = converter

override def end(): Unit = {
converter.updater.end()
currentArray += currentElement
}
override def end(): Unit = currentArray += currentElement

override def start(): Unit = {
converter.updater.start()
currentElement = null
}
override def start(): Unit = currentElement = null
}
}

Expand Down