From 60199ec705ad7ea2e5fee3f55935630e9ed478bf Mon Sep 17 00:00:00 2001 From: xiepengjie Date: Mon, 6 Jul 2020 17:46:13 +0800 Subject: [PATCH 1/4] [SPARK-32192][SQL] Print column name when throws ClassCastException --- .../apache/spark/sql/hive/TableReader.scala | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index 4d18eb628941..bed793d46c24 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -47,6 +47,8 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{SerializableConfiguration, Utils} +import scala.util.{Failure, Success, Try} + /** * A trait for subclasses that handle table scans. */ @@ -486,13 +488,20 @@ private[hive] object HadoopTableReader extends HiveInspectors with Logging { var i = 0 val length = fieldRefs.length while (i < length) { - val fieldValue = soi.getStructFieldData(raw, fieldRefs(i)) - if (fieldValue == null) { - mutableRow.setNullAt(fieldOrdinals(i)) - } else { - unwrappers(i)(fieldValue, mutableRow, fieldOrdinals(i)) + Try { + val fieldValue = soi.getStructFieldData(raw, fieldRefs(i)) + if (fieldValue == null) { + mutableRow.setNullAt(fieldOrdinals(i)) + } else { + unwrappers(i)(fieldValue, mutableRow, fieldOrdinals(i)) + } + i += 1 + } match { + case Failure(ex) => + logError(s"Exception thrown in field <${fieldRefs(i).getFieldName}>") + throw ex + case Success(value) => value } - i += 1 } mutableRow: InternalRow From a535b4d25c99bcf7717e8f45d462cb58e32ed438 Mon Sep 17 00:00:00 2001 From: xiepengjie Date: Mon, 6 Jul 2020 18:38:41 +0800 Subject: [PATCH 2/4] fix import style --- .../src/main/scala/org/apache/spark/sql/hive/TableReader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index bed793d46c24..194a2a6d35c6 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.hive import java.util.Properties import scala.collection.JavaConverters._ +import scala.util.{Failure, Success, Try} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{Path, PathFilter} @@ -47,7 +48,6 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{SerializableConfiguration, Utils} -import scala.util.{Failure, Success, Try} /** * A trait for subclasses that handle table scans. From 3e519305473cc0017d4446b80cc6b927f17ff847 Mon Sep 17 00:00:00 2001 From: xiepengjie Date: Tue, 7 Jul 2020 11:13:04 +0800 Subject: [PATCH 3/4] remove the unnecessary change & replace Try-match with try-catch --- .../scala/org/apache/spark/sql/hive/TableReader.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index 194a2a6d35c6..b02380c1533a 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -48,7 +48,6 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.unsafe.types.UTF8String import org.apache.spark.util.{SerializableConfiguration, Utils} - /** * A trait for subclasses that handle table scans. */ @@ -488,7 +487,7 @@ private[hive] object HadoopTableReader extends HiveInspectors with Logging { var i = 0 val length = fieldRefs.length while (i < length) { - Try { + try { val fieldValue = soi.getStructFieldData(raw, fieldRefs(i)) if (fieldValue == null) { mutableRow.setNullAt(fieldOrdinals(i)) @@ -496,11 +495,10 @@ private[hive] object HadoopTableReader extends HiveInspectors with Logging { unwrappers(i)(fieldValue, mutableRow, fieldOrdinals(i)) } i += 1 - } match { - case Failure(ex) => + } catch { + case ex: Throwable => logError(s"Exception thrown in field <${fieldRefs(i).getFieldName}>") throw ex - case Success(value) => value } } From 018c98180afd105c92e3f616121627711dd5336c Mon Sep 17 00:00:00 2001 From: xiepengjie Date: Wed, 8 Jul 2020 09:22:07 +0800 Subject: [PATCH 4/4] remove unnecessary import --- .../src/main/scala/org/apache/spark/sql/hive/TableReader.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala index b02380c1533a..8ddb979d7713 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.hive import java.util.Properties import scala.collection.JavaConverters._ -import scala.util.{Failure, Success, Try} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{Path, PathFilter}