File tree Expand file tree Collapse file tree 3 files changed +10
-6
lines changed
core/src/main/scala/org/apache/spark/sql/parquet
hive/src/test/scala/org/apache/spark/sql/parquet Expand file tree Collapse file tree 3 files changed +10
-6
lines changed Original file line number Diff line number Diff line change @@ -90,7 +90,7 @@ trait ParquetTest {
9090 (f : String => Unit ): Unit = {
9191 import sqlContext .implicits ._
9292 withTempPath { file =>
93- sparkContext.parallelize(data, 3 ).saveAsParquetFile(file.getCanonicalPath)
93+ sparkContext.parallelize(data).saveAsParquetFile(file.getCanonicalPath)
9494 f(file.getCanonicalPath)
9595 }
9696 }
Original file line number Diff line number Diff line change @@ -482,12 +482,11 @@ object ParquetRelation2 {
482482 * Parses a single partition, returns column names and values of each partition column. For
483483 * example, given:
484484 * {{{
485- * basePath = hdfs://<host>:<port>/path/to/partition
486- * partitionPath = hdfs://<host>:<port>/path/to/partition/a=42/b=hello/c=3.14
485+ * path = hdfs://<host>:<port>/path/to/partition/a=42/b=hello/c=3.14
487486 * }}}
488487 * it returns:
489488 * {{{
490- * PartitionDesc (
489+ * PartitionValues (
491490 * Seq("a", "b", "c"),
492491 * Seq(
493492 * Literal(42, IntegerType),
@@ -587,7 +586,12 @@ object ParquetRelation2 {
587586 * types.
588587 */
589588 private def resolveTypeConflicts (literals : Seq [Literal ]): Seq [Literal ] = {
590- val desiredType = literals.map(_.dataType).maxBy(upCastingOrder.indexOf(_))
589+ val desiredType = {
590+ val topType = literals.map(_.dataType).maxBy(upCastingOrder.indexOf(_))
591+ // Falls back to string if all values of this column are null or empty string
592+ if (topType == NullType ) StringType else topType
593+ }
594+
591595 literals.map { case l @ Literal (_, dataType) =>
592596 Literal (Cast (l, desiredType).eval(), desiredType)
593597 }
Original file line number Diff line number Diff line change @@ -245,7 +245,7 @@ abstract class ParquetPartitioningTest extends QueryTest with BeforeAndAfterAll
245245 Row (10 ))
246246 }
247247
248- test(s " $prefix: non-existant partition $table" ) {
248+ test(s " $prefix: non-existent partition $table" ) {
249249 checkAnswer(
250250 sql(s " SELECT COUNT(*) FROM $table WHERE p = 1000 " ),
251251 Row (0 ))
You can’t perform that action at this time.
0 commit comments