Skip to content

Commit 9968fba

Browse files
committed
Tests the data type conversion code paths
1 parent 8af1bf1 commit 9968fba

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

sql/hive/src/test/scala/org/apache/spark/sql/sources/SimpleTextRelation.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat, TextOutputForma
2727
import org.apache.hadoop.mapreduce.{Job, RecordWriter, TaskAttemptContext}
2828

2929
import org.apache.spark.rdd.RDD
30+
import org.apache.spark.sql.catalyst.CatalystTypeConverters
3031
import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
3132
import org.apache.spark.sql.types.{DataType, StructType}
3233
import org.apache.spark.sql.{Row, SQLContext}
@@ -108,7 +109,10 @@ class SimpleTextRelation(
108109

109110
sparkContext.textFile(inputStatuses.map(_.getPath).mkString(",")).map { record =>
110111
Row(record.split(",").zip(fields).map { case (value, dataType) =>
111-
Cast(Literal(value), dataType).eval()
112+
// `Cast`ed values are always of Catalyst types (i.e. UTF8String instead of String, etc.)
113+
val catalystValue = Cast(Literal(value), dataType).eval()
114+
// Here we're converting Catalyst values to Scala values to test `needsConversion`
115+
CatalystTypeConverters.convertToScala(catalystValue, dataType)
112116
}: _*)
113117
}
114118
}

0 commit comments

Comments
 (0)