From afd34528d37b575d84f9380b3475f509c263e6e3 Mon Sep 17 00:00:00 2001 From: Josh Rosen Date: Tue, 19 Jan 2016 23:48:12 -0800 Subject: [PATCH 1/2] Use SparkHadoopUtil.getConfigurationFromJobContext() in SpecificParquetRecordReaderBase --- .../datasources/parquet/SpecificParquetRecordReaderBase.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java index 2ed30c1f5a8d9..e5f9ce8cd2fe4 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java @@ -57,6 +57,8 @@ import org.apache.parquet.hadoop.util.ConfigurationUtil; import org.apache.parquet.schema.MessageType; +import org.apache.spark.deploy.SparkHadoopUtil; + /** * Base class for custom RecordReaaders for Parquet that directly materialize to `T`. * This class handles computing row groups, filtering on them, setting up the column readers, @@ -81,7 +83,8 @@ public abstract class SpecificParquetRecordReaderBase extends RecordReader Date: Wed, 20 Jan 2016 01:31:49 -0800 Subject: [PATCH 2/2] Fix compilation --- .../datasources/parquet/SpecificParquetRecordReaderBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java index e5f9ce8cd2fe4..bf9799bd130a6 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/SpecificParquetRecordReaderBase.java @@ -84,7 +84,7 @@ public abstract class SpecificParquetRecordReaderBase extends RecordReader