From 6d6ae340f3fb641a96ef9ec2c561d9c6f6939434 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Fri, 3 Apr 2015 11:19:04 -0700 Subject: [PATCH 1/2] Use path.makeQualified. --- .../main/scala/org/apache/spark/sql/parquet/newParquet.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala index 583bac42fdcce..1259e78bdbe6a 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala @@ -268,7 +268,8 @@ private[sql] case class ParquetRelation2( // containing Parquet files (e.g. partitioned Parquet table). val baseStatuses = paths.distinct.map { p => val fs = FileSystem.get(URI.create(p), sparkContext.hadoopConfiguration) - val qualified = fs.makeQualified(new Path(p)) + val path = new Path(p) + val qualified = path.makeQualified(fs) if (!fs.exists(qualified) && maybeSchema.isDefined) { fs.mkdirs(qualified) From 849603b283bb355b898976694940c97817e25aa7 Mon Sep 17 00:00:00 2001 From: Yin Huai Date: Fri, 3 Apr 2015 11:21:35 -0700 Subject: [PATCH 2/2] Not use deprecated method. --- .../main/scala/org/apache/spark/sql/parquet/newParquet.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala index 1259e78bdbe6a..0dce3623a66df 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala @@ -269,7 +269,7 @@ private[sql] case class ParquetRelation2( val baseStatuses = paths.distinct.map { p => val fs = FileSystem.get(URI.create(p), sparkContext.hadoopConfiguration) val path = new Path(p) - val qualified = path.makeQualified(fs) + val qualified = path.makeQualified(fs.getUri, fs.getWorkingDirectory) if (!fs.exists(qualified) && maybeSchema.isDefined) { fs.mkdirs(qualified)