Skip to content

Commit 15cd105

Browse files
committed
Fixes SPARK-10136 in a more proper way
1 parent dcfe0c5 commit 15cd105

File tree

1 file changed

+7
-10
lines changed

1 file changed

+7
-10
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/CatalystRowConverter.scala

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -415,8 +415,9 @@ private[parquet] class CatalystRowConverter(
415415
private val elementConverter: Converter = {
416416
val repeatedType = parquetSchema.getType(0)
417417
val elementType = catalystSchema.elementType
418+
val parentName = parquetSchema.getName
418419

419-
if (isElementType(repeatedType, elementType)) {
420+
if (isElementType(repeatedType, elementType, parentName)) {
420421
newConverter(repeatedType, elementType, new ParentContainerUpdater {
421422
override def set(value: Any): Unit = currentArray += value
422423
})
@@ -453,10 +454,12 @@ private[parquet] class CatalystRowConverter(
453454
* @see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
454455
*/
455456
// scalastyle:on
456-
private def isElementType(parquetRepeatedType: Type, catalystElementType: DataType): Boolean = {
457+
private def isElementType(
458+
parquetRepeatedType: Type, catalystElementType: DataType, parentName: String): Boolean = {
457459
(parquetRepeatedType, catalystElementType) match {
458460
case (t: PrimitiveType, _) => true
459461
case (t: GroupType, _) if t.getFieldCount > 1 => true
462+
case (t, _) if t.getName == "array" || t.getName == s"${parentName}_tuple" => true
460463
case (t: GroupType, StructType(Array(f))) if f.name == t.getFieldName(0) => true
461464
case _ => false
462465
}
@@ -474,15 +477,9 @@ private[parquet] class CatalystRowConverter(
474477

475478
override def getConverter(fieldIndex: Int): Converter = converter
476479

477-
override def end(): Unit = {
478-
converter.updater.end()
479-
currentArray += currentElement
480-
}
480+
override def end(): Unit = currentArray += currentElement
481481

482-
override def start(): Unit = {
483-
converter.updater.start()
484-
currentElement = null
485-
}
482+
override def start(): Unit = currentElement = null
486483
}
487484
}
488485

0 commit comments

Comments
 (0)