From 3b27af36f82580c2171df965140c9a14e62fd5f0 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Fri, 30 Jan 2015 21:48:21 +0800
Subject: [PATCH 01/13] SPARK-5498:fix bug when query the data when partition
 schema does not match table schema

---
 .../apache/spark/sql/hive/TableReader.scala   | 44 ++++++++++++++++---
 .../sql/hive/InsertIntoHiveTableSuite.scala   | 15 +++++++
 2 files changed, 54 insertions(+), 5 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index c368715f7c6f5..8f955b699981e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities
 import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Table => HiveTable}
 import org.apache.hadoop.hive.ql.plan.{PlanUtils, TableDesc}
 import org.apache.hadoop.hive.serde2.Deserializer
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector
+import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
 import org.apache.hadoop.io.Writable
 import org.apache.hadoop.mapred.{FileInputFormat, InputFormat, JobConf}
@@ -188,9 +188,13 @@ class HadoopTableReader(
         val hconf = broadcastedHiveConf.value.value
         val deserializer = localDeserializer.newInstance()
         deserializer.initialize(hconf, partProps)
+        //get the table deserializer 
+        val tableSerDe = tableDesc.getDeserializerClass.newInstance()
+        tableSerDe.initialize(hconf, tableDesc.getProperties)
 
         // fill the non partition key attributes
-        HadoopTableReader.fillObject(iter, deserializer, nonPartitionKeyAttrs, mutableRow)
+        HadoopTableReader.fillObject(iter, deserializer, nonPartitionKeyAttrs,
+          mutableRow, Some(tableSerDe))
       }
     }.toSeq
 
@@ -264,15 +268,31 @@ private[hive] object HadoopTableReader extends HiveInspectors {
    * @param nonPartitionKeyAttrs Attributes that should be filled together with their corresponding
    *                             positions in the output schema
    * @param mutableRow A reusable `MutableRow` that should be filled
+   * @param convertdeserializer The `Deserializer` covert the `deserializer`                
    * @return An `Iterator[Row]` transformed from `iterator`
    */
   def fillObject(
       iterator: Iterator[Writable],
       deserializer: Deserializer,
       nonPartitionKeyAttrs: Seq[(Attribute, Int)],
-      mutableRow: MutableRow): Iterator[Row] = {
+      mutableRow: MutableRow,
+      convertdeserializer: Option[Deserializer] = None): Iterator[Row] = {
+
+    val soi = convertdeserializer match {
+      case Some(convert) =>
+        //check need to convert
+        if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
+          deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
+        }
+        else {
+          ObjectInspectorConverters.getConvertedOI(
+            deserializer.getObjectInspector(), convert.getObjectInspector(), true)
+            .asInstanceOf[StructObjectInspector]
+        }
+      case None =>
+        deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
+    }
 
-    val soi = deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
     val (fieldRefs, fieldOrdinals) = nonPartitionKeyAttrs.map { case (attr, ordinal) =>
       soi.getStructFieldRef(attr.name) -> ordinal
     }.unzip
@@ -315,9 +335,23 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       }
     }
 
+    val partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
+      deserializer.getObjectInspector, soi)
+
     // Map each tuple to a row object
     iterator.map { value =>
-      val raw = deserializer.deserialize(value)
+      val raw = convertdeserializer match {
+        case Some(convert) =>          
+          if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
+            deserializer.deserialize(value)
+          }
+          //If partition schema does not match table schema, update the row to match  
+          else {
+            partTblObjectInspectorConverter.convert(deserializer.deserialize(value))
+          }
+        case None =>
+          deserializer.deserialize(value)
+      }
       var i = 0
       while (i < fieldRefs.length) {
         val fieldValue = soi.getStructFieldData(raw, fieldRefs(i))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
index 4dd96bd5a1b77..de87da1b7f82e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
@@ -172,4 +172,19 @@ class InsertIntoHiveTableSuite extends QueryTest {
 
     sql("DROP TABLE hiveTableWithStructValue")
   }
+  
+  test("SPARK-5498:partition schema does not match table schema"){
+    val testData = TestHive.sparkContext.parallelize(
+      (1 to 10).map(i => TestData(i, i.toString)))
+    testData.registerTempTable("testData")
+    val tmpDir = Files.createTempDir()
+    sql(s"CREATE TABLE table_with_partition(key int,value string) PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
+    sql("INSERT OVERWRITE TABLE table_with_partition  partition (ds='1') SELECT key,value FROM testData")
+    sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
+    checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
+      testData.toSchemaRDD.collect.toSeq
+    )
+    sql("DROP TABLE table_with_partition")
+    
+  }
 }

From 10744cac4cc014216241fc97c6495924cc51fdbc Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Mon, 2 Feb 2015 10:10:36 +0800
Subject: [PATCH 02/13] Insert a space after the start of the comment

---
 .../main/scala/org/apache/spark/sql/hive/TableReader.scala  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index 8f955b699981e..72edb89ad3358 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -188,7 +188,7 @@ class HadoopTableReader(
         val hconf = broadcastedHiveConf.value.value
         val deserializer = localDeserializer.newInstance()
         deserializer.initialize(hconf, partProps)
-        //get the table deserializer 
+        // get the table deserializer
         val tableSerDe = tableDesc.getDeserializerClass.newInstance()
         tableSerDe.initialize(hconf, tableDesc.getProperties)
 
@@ -280,7 +280,7 @@ private[hive] object HadoopTableReader extends HiveInspectors {
 
     val soi = convertdeserializer match {
       case Some(convert) =>
-        //check need to convert
+        // check need to convert
         if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
           deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
         }
@@ -345,7 +345,7 @@ private[hive] object HadoopTableReader extends HiveInspectors {
           if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
             deserializer.deserialize(value)
           }
-          //If partition schema does not match table schema, update the row to match  
+          // If partition schema does not match table schema, update the row to match
           else {
             partTblObjectInspectorConverter.convert(deserializer.deserialize(value))
           }

From b1527d58349ccdc0b986705b93d7658822211994 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Mon, 2 Feb 2015 10:49:19 +0800
Subject: [PATCH 03/13] fix type mismatch

---
 .../main/scala/org/apache/spark/sql/hive/TableReader.scala    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index 72edb89ad3358..d76ac40302ff3 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -286,8 +286,8 @@ private[hive] object HadoopTableReader extends HiveInspectors {
         }
         else {
           ObjectInspectorConverters.getConvertedOI(
-            deserializer.getObjectInspector(), convert.getObjectInspector(), true)
-            .asInstanceOf[StructObjectInspector]
+            deserializer.getObjectInspector(), convert.getObjectInspector(),
+            new java.lang.Boolean(true)).asInstanceOf[StructObjectInspector]
         }
       case None =>
         deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]

From afc7da53be4b7bcb9cd5ce8d72b6855544b96596 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Mon, 2 Feb 2015 13:00:38 +0800
Subject: [PATCH 04/13] make getConvertedOI compatible between 0.12.0 and
 0.13.1

---
 .../main/scala/org/apache/spark/sql/hive/TableReader.scala | 6 +++---
 .../src/main/scala/org/apache/spark/sql/hive/Shim12.scala  | 7 ++++++-
 .../src/main/scala/org/apache/spark/sql/hive/Shim13.scala  | 7 ++++++-
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index d76ac40302ff3..c54ce886a7f35 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -285,9 +285,9 @@ private[hive] object HadoopTableReader extends HiveInspectors {
           deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
         }
         else {
-          ObjectInspectorConverters.getConvertedOI(
-            deserializer.getObjectInspector(), convert.getObjectInspector(),
-            new java.lang.Boolean(true)).asInstanceOf[StructObjectInspector]
+          HiveShim.getConvertedOI(
+            deserializer.getObjectInspector(), 
+            convert.getObjectInspector()).asInstanceOf[StructObjectInspector]
         }
       case None =>
         deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index c0b7741bc3e53..481914964d931 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -34,7 +34,7 @@ import org.apache.hadoop.hive.ql.plan.{CreateTableDesc, FileSinkDesc, TableDesc}
 import org.apache.hadoop.hive.ql.processors._
 import org.apache.hadoop.hive.ql.stats.StatsSetupConst
 import org.apache.hadoop.hive.serde2.{ColumnProjectionUtils, Deserializer, io => hiveIo}
-import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, PrimitiveObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, ObjectInspector, PrimitiveObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.{HiveDecimalObjectInspector, PrimitiveObjectInspectorFactory}
 import org.apache.hadoop.hive.serde2.typeinfo.{TypeInfo, TypeInfoFactory}
@@ -241,6 +241,11 @@ private[hive] object HiveShim {
       Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue())
     }
   }
+
+  // make getConvertedOI compatible between 0.12.0 and 0.13.1
+  def getConvertedOI(inputOI: ObjectInspector, outputOI: ObjectInspector): ObjectInspector = {
+    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, new java.lang.Boolean(true))
+  }
 }
 
 class ShimFileSinkDesc(var dir: String, var tableInfo: TableDesc, var compressed: Boolean)
diff --git a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
index c04cda7bf1537..9bbfc654ae50e 100644
--- a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
+++ b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
@@ -36,7 +36,7 @@ import org.apache.hadoop.hive.ql.plan.{CreateTableDesc, FileSinkDesc, TableDesc}
 import org.apache.hadoop.hive.ql.processors.CommandProcessorFactory
 import org.apache.hadoop.hive.serde2.typeinfo.{TypeInfo, DecimalTypeInfo, TypeInfoFactory}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.{HiveDecimalObjectInspector, PrimitiveObjectInspectorFactory}
-import org.apache.hadoop.hive.serde2.objectinspector.{PrimitiveObjectInspector, ObjectInspector}
+import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, PrimitiveObjectInspector, ObjectInspector}
 import org.apache.hadoop.hive.serde2.{Deserializer, ColumnProjectionUtils}
 import org.apache.hadoop.hive.serde2.{io => hiveIo}
 import org.apache.hadoop.{io => hadoopIo}
@@ -395,6 +395,11 @@ private[hive] object HiveShim {
       Decimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue(), hdoi.precision(), hdoi.scale())
     }
   }
+
+  // make getConvertedOI compatible between 0.12.0 and 0.13.1
+  def getConvertedOI(inputOI: ObjectInspector, outputOI: ObjectInspector): ObjectInspector = {
+    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI)
+  }
 }
 
 /*

From 63d170a633304c63ff55d2be399442d9784cfe8d Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Tue, 3 Feb 2015 12:33:51 +0800
Subject: [PATCH 05/13] fix compile problem

---
 .../scala/org/apache/spark/sql/hive/Shim12.scala    | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index dd137900e310d..790f828c82a45 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -61,10 +61,10 @@ private[hive] object HiveShim {
   val version = "0.12.0"
 
   def getTableDesc(
-    serdeClass: Class[_ <: Deserializer],
-    inputFormatClass: Class[_ <: InputFormat[_, _]],
-    outputFormatClass: Class[_],
-    properties: Properties) = {
+                    serdeClass: Class[_ <: Deserializer],
+                    inputFormatClass: Class[_ <: InputFormat[_, _]],
+                    outputFormatClass: Class[_],
+                    properties: Properties) = {
     new TableDesc(serdeClass, inputFormatClass, outputFormatClass, properties)
   }
 
@@ -187,7 +187,7 @@ private[hive] object HiveShim {
 
   def getStatsSetupConstRawDataSize = StatsSetupConst.RAW_DATA_SIZE
 
-  def createDefaultDBIfNeeded(context: HiveContext) = {  }
+  def createDefaultDBIfNeeded(context: HiveContext) = {}
 
   def getCommandProcessor(cmd: Array[String], conf: HiveConf) = {
     CommandProcessorFactory.get(cmd(0), conf)
@@ -208,7 +208,7 @@ private[hive] object HiveShim {
 
   def getDataLocationPath(p: Partition) = p.getPartitionPath
 
-  def getAllPartitionsOf(client: Hive, tbl: Table) =  client.getAllPartitionsForPruner(tbl)
+  def getAllPartitionsOf(client: Hive, tbl: Table) = client.getAllPartitionsForPruner(tbl)
 
   def compatibilityBlackList = Seq(
     "decimal_.*",
@@ -250,6 +250,7 @@ private[hive] object HiveShim {
   def prepareWritable(w: Writable): Writable = {
     w
   }
+}
 
 class ShimFileSinkDesc(var dir: String, var tableInfo: TableDesc, var compressed: Boolean)
   extends FileSinkDesc(dir, tableInfo, compressed) {

From 12d800d0904fba3564caba7613a35caa6fc812ce Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Tue, 3 Feb 2015 13:50:35 +0800
Subject: [PATCH 06/13] fix code style

---
 .../src/main/scala/org/apache/spark/sql/hive/Shim12.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index 790f828c82a45..762eae57ff86d 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -61,10 +61,10 @@ private[hive] object HiveShim {
   val version = "0.12.0"
 
   def getTableDesc(
-                    serdeClass: Class[_ <: Deserializer],
-                    inputFormatClass: Class[_ <: InputFormat[_, _]],
-                    outputFormatClass: Class[_],
-                    properties: Properties) = {
+    serdeClass: Class[_ <: Deserializer],
+    inputFormatClass: Class[_ <: InputFormat[_, _]],
+    outputFormatClass: Class[_],
+    properties: Properties) = {
     new TableDesc(serdeClass, inputFormatClass, outputFormatClass, properties)
   }
 

From 2a91a87d27e31497ebefd2ad80531c0017fafbc5 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Thu, 5 Feb 2015 11:25:36 +0800
Subject: [PATCH 07/13] add more test case and clean the code

---
 .../apache/spark/sql/hive/TableReader.scala   | 18 ++++-------
 .../sql/hive/InsertIntoHiveTableSuite.scala   | 30 +++++++++++++++++--
 .../org/apache/spark/sql/hive/Shim12.scala    |  7 +++--
 .../org/apache/spark/sql/hive/Shim13.scala    |  1 +
 4 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index c54ce886a7f35..a6cfd978b5f62 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -335,23 +335,17 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       }
     }
 
+    /**
+     * when the soi and deserializer.getObjectInspector is equal,
+     * we will get `IdentityConverter`,which mean it won't convert the
+     * value when schema match
+     */
     val partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
       deserializer.getObjectInspector, soi)
 
     // Map each tuple to a row object
     iterator.map { value =>
-      val raw = convertdeserializer match {
-        case Some(convert) =>          
-          if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
-            deserializer.deserialize(value)
-          }
-          // If partition schema does not match table schema, update the row to match
-          else {
-            partTblObjectInspectorConverter.convert(deserializer.deserialize(value))
-          }
-        case None =>
-          deserializer.deserialize(value)
-      }
+      val raw = partTblObjectInspectorConverter.convert(deserializer.deserialize(value))
       var i = 0
       while (i < fieldRefs.length) {
         val fieldValue = soi.getStructFieldData(raw, fieldRefs(i))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
index de87da1b7f82e..33e859427a4b0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
@@ -29,6 +29,8 @@ import org.apache.spark.sql.hive.test.TestHive._
 
 case class TestData(key: Int, value: String)
 
+case class ThreeCloumntable(key: Int, value: String, key1: String)
+
 class InsertIntoHiveTableSuite extends QueryTest {
   val testData = TestHive.sparkContext.parallelize(
     (1 to 100).map(i => TestData(i, i.toString)))
@@ -172,19 +174,43 @@ class InsertIntoHiveTableSuite extends QueryTest {
 
     sql("DROP TABLE hiveTableWithStructValue")
   }
-  
+
   test("SPARK-5498:partition schema does not match table schema"){
     val testData = TestHive.sparkContext.parallelize(
       (1 to 10).map(i => TestData(i, i.toString)))
     testData.registerTempTable("testData")
+
+    val testDatawithNull = TestHive.sparkContext.parallelize(
+      (1 to 10).map(i => ThreeCloumntable(i, i.toString,null)))
+
     val tmpDir = Files.createTempDir()
     sql(s"CREATE TABLE table_with_partition(key int,value string) PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
     sql("INSERT OVERWRITE TABLE table_with_partition  partition (ds='1') SELECT key,value FROM testData")
+
+    //test schema is the same
     sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
     checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
       testData.toSchemaRDD.collect.toSeq
     )
-    sql("DROP TABLE table_with_partition")
     
+    // test difference type of field
+    sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
+    checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
+      testData.toSchemaRDD.collect.toSeq
+    )
+
+    // add column to table
+    sql("ALTER TABLE table_with_partition ADD COLUMNS(key1 string)")
+    checkAnswer(sql("select key,value,key1 from table_with_partition where ds='1' "),
+      testDatawithNull.toSchemaRDD.collect.toSeq
+    )
+
+    // change column name to table
+    sql("ALTER TABLE table_with_partition CHANGE COLUMN key keynew BIGINT")
+    checkAnswer(sql("select keynew,value from table_with_partition where ds='1' "),
+      testData.toSchemaRDD.collect.toSeq
+    )
+
+    sql("DROP TABLE table_with_partition")
   }
 }
diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index 762eae57ff86d..4fe8e8621f7b8 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -243,8 +243,11 @@ private[hive] object HiveShim {
   }
 
   // make getConvertedOI compatible between 0.12.0 and 0.13.1
-  def getConvertedOI(inputOI: ObjectInspector, outputOI: ObjectInspector): ObjectInspector = {
-    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, new java.lang.Boolean(true))
+  def getConvertedOI(inputOI: ObjectInspector,
+                     outputOI: ObjectInspector,
+                     equalsCheck: java.lang.Boolean =
+                     new java.lang.Boolean(true)): ObjectInspector = {
+    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, equalsCheck)
   }
 
   def prepareWritable(w: Writable): Writable = {
diff --git a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
index ff61479b33dda..55041417ecc17 100644
--- a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
+++ b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.hive
 
+import java.util
 import java.util.{ArrayList => JArrayList}
 import java.util.Properties
 import java.rmi.server.UID

From c879aa1d68ef41ca95ce2084326a6f14a00deb4f Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Sun, 8 Feb 2015 02:55:15 +0800
Subject: [PATCH 08/13] clean the code

---
 .../apache/spark/sql/hive/TableReader.scala   | 47 +++++++------------
 .../sql/hive/InsertIntoHiveTableSuite.scala   |  2 +-
 .../org/apache/spark/sql/hive/Shim12.scala    |  7 +--
 .../org/apache/spark/sql/hive/Shim13.scala    |  1 -
 4 files changed, 21 insertions(+), 36 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index a6cfd978b5f62..d618ec5298526 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities
 import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Table => HiveTable}
 import org.apache.hadoop.hive.ql.plan.{PlanUtils, TableDesc}
 import org.apache.hadoop.hive.serde2.Deserializer
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.IdentityConverter
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
 import org.apache.hadoop.io.Writable
@@ -115,7 +116,7 @@ class HadoopTableReader(
       val hconf = broadcastedHiveConf.value.value
       val deserializer = deserializerClass.newInstance()
       deserializer.initialize(hconf, tableDesc.getProperties)
-      HadoopTableReader.fillObject(iter, deserializer, attrsWithIndex, mutableRow)
+      HadoopTableReader.fillObject(iter, deserializer, attrsWithIndex, mutableRow, deserializer)
     }
 
     deserializedHadoopRDD
@@ -194,7 +195,7 @@ class HadoopTableReader(
 
         // fill the non partition key attributes
         HadoopTableReader.fillObject(iter, deserializer, nonPartitionKeyAttrs,
-          mutableRow, Some(tableSerDe))
+          mutableRow, tableSerDe)
       }
     }.toSeq
 
@@ -264,37 +265,27 @@ private[hive] object HadoopTableReader extends HiveInspectors {
    * Transform all given raw `Writable`s into `Row`s.
    *
    * @param iterator Iterator of all `Writable`s to be transformed
-   * @param deserializer The `Deserializer` associated with the input `Writable`
+   * @param rawDeser The `Deserializer` associated with the input `Writable`
    * @param nonPartitionKeyAttrs Attributes that should be filled together with their corresponding
    *                             positions in the output schema
    * @param mutableRow A reusable `MutableRow` that should be filled
-   * @param convertdeserializer The `Deserializer` covert the `deserializer`                
+   * @param tableDeser Table Deserializer
    * @return An `Iterator[Row]` transformed from `iterator`
    */
   def fillObject(
       iterator: Iterator[Writable],
-      deserializer: Deserializer,
+      rawDeser: Deserializer,
       nonPartitionKeyAttrs: Seq[(Attribute, Int)],
       mutableRow: MutableRow,
-      convertdeserializer: Option[Deserializer] = None): Iterator[Row] = {
+      tableDeser: Deserializer): Iterator[Row] = {
 
-    val soi = convertdeserializer match {
-      case Some(convert) =>
-        // check need to convert
-        if (deserializer.getObjectInspector.equals(convert.getObjectInspector)) {
-          deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
-        }
-        else {
-          HiveShim.getConvertedOI(
-            deserializer.getObjectInspector(), 
-            convert.getObjectInspector()).asInstanceOf[StructObjectInspector]
-        }
-      case None =>
-        deserializer.getObjectInspector().asInstanceOf[StructObjectInspector]
-    }
+    val soi = HiveShim.getConvertedOI(
+      rawDeser.getObjectInspector, tableDeser.getObjectInspector).asInstanceOf[StructObjectInspector]
+
+    val inputFields = soi.getAllStructFieldRefs
 
     val (fieldRefs, fieldOrdinals) = nonPartitionKeyAttrs.map { case (attr, ordinal) =>
-      soi.getStructFieldRef(attr.name) -> ordinal
+      (inputFields.get(ordinal), ordinal)
     }.unzip
 
     // Builds specific unwrappers ahead of time according to object inspector types to avoid pattern
@@ -335,17 +326,15 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       }
     }
 
-    /**
-     * when the soi and deserializer.getObjectInspector is equal,
-     * we will get `IdentityConverter`,which mean it won't convert the
-     * value when schema match
-     */
-    val partTblObjectInspectorConverter = ObjectInspectorConverters.getConverter(
-      deserializer.getObjectInspector, soi)
+    val converter = if (rawDeser == tableDeser) {
+      new IdentityConverter
+    } else {
+      ObjectInspectorConverters.getConverter(rawDeser.getObjectInspector, soi)
+    }
 
     // Map each tuple to a row object
     iterator.map { value =>
-      val raw = partTblObjectInspectorConverter.convert(deserializer.deserialize(value))
+      val raw = converter.convert(rawDeser.deserialize(value))
       var i = 0
       while (i < fieldRefs.length) {
         val fieldValue = soi.getStructFieldData(raw, fieldRefs(i))
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
index 33e859427a4b0..5ddde890da32a 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
@@ -187,7 +187,7 @@ class InsertIntoHiveTableSuite extends QueryTest {
     sql(s"CREATE TABLE table_with_partition(key int,value string) PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
     sql("INSERT OVERWRITE TABLE table_with_partition  partition (ds='1') SELECT key,value FROM testData")
 
-    //test schema is the same
+    // test schema the same between partition and table
     sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
     checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
       testData.toSchemaRDD.collect.toSeq
diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index 4fe8e8621f7b8..d7a4b509b1468 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -242,12 +242,9 @@ private[hive] object HiveShim {
     }
   }
 
-  // make getConvertedOI compatible between 0.12.0 and 0.13.1
   def getConvertedOI(inputOI: ObjectInspector,
-                     outputOI: ObjectInspector,
-                     equalsCheck: java.lang.Boolean =
-                     new java.lang.Boolean(true)): ObjectInspector = {
-    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, equalsCheck)
+                     outputOI: ObjectInspector): ObjectInspector = {
+    ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, true)
   }
 
   def prepareWritable(w: Writable): Writable = {
diff --git a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
index 55041417ecc17..3f2b6859e61ce 100644
--- a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
+++ b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
@@ -399,7 +399,6 @@ private[hive] object HiveShim {
     }
   }
 
-  // make getConvertedOI compatible between 0.12.0 and 0.13.1
   def getConvertedOI(inputOI: ObjectInspector, outputOI: ObjectInspector): ObjectInspector = {
     ObjectInspectorConverters.getConvertedOI(inputOI, outputOI)
   }

From 0549759b4ed1a672759bd2553553899abcc48453 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Sun, 8 Feb 2015 03:13:51 +0800
Subject: [PATCH 09/13] fix code style

---
 hs_err_pid6969.log                            | 352 ++++++++++++++++++
 hs_err_pid7002.log                            | 352 ++++++++++++++++++
 .../apache/spark/sql/hive/TableReader.scala   |   9 +-
 3 files changed, 710 insertions(+), 3 deletions(-)
 create mode 100644 hs_err_pid6969.log
 create mode 100644 hs_err_pid7002.log

diff --git a/hs_err_pid6969.log b/hs_err_pid6969.log
new file mode 100644
index 0000000000000..176ac9a4af8be
--- /dev/null
+++ b/hs_err_pid6969.log
@@ -0,0 +1,352 @@
+#
+# There is insufficient memory for the Java Runtime Environment to continue.
+# Native memory allocation (malloc) failed to allocate 1431830528 bytes for committing reserved memory.
+# Possible reasons:
+#   The system is out of physical RAM or swap space
+#   In 32 bit mode, the process size limit was hit
+# Possible solutions:
+#   Reduce memory load on the system
+#   Increase physical memory or swap space
+#   Check if swap backing store is full
+#   Use 64 bit Java on a 64 bit OS
+#   Decrease Java heap size (-Xmx/-Xms)
+#   Decrease number of Java threads
+#   Decrease Java thread stack sizes (-Xss)
+#   Set larger code cache with -XX:ReservedCodeCacheSize=
+# This output file may be truncated or incomplete.
+#
+#  Out of Memory Error (os_linux.cpp:2745), pid=6969, tid=139930463151872
+#
+# JRE version:  (7.0_71-b14) (build )
+# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.71-b01 mixed mode linux-amd64 compressed oops)
+# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
+#
+
+---------------  T H R E A D  ---------------
+
+Current thread (0x00007f441000a000):  JavaThread "Unknown thread" [_thread_in_vm, id=7000, stack(0x00007f44197ca000,0x00007f44198cb000)]
+
+Stack: [0x00007f44197ca000,0x00007f44198cb000],  sp=0x00007f44198c9230,  free space=1020k
+Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
+V  [libjvm.so+0x9a0a1a]  VMError::report_and_die()+0x2ea
+V  [libjvm.so+0x4974bb]  report_vm_out_of_memory(char const*, int, unsigned long, char const*)+0x9b
+V  [libjvm.so+0x81e4de]  os::Linux::commit_memory_impl(char*, unsigned long, bool)+0xfe
+V  [libjvm.so+0x81ea29]  os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x29
+V  [libjvm.so+0x8187da]  os::commit_memory(char*, unsigned long, unsigned long, bool)+0x2a
+V  [libjvm.so+0x88b2e3]  PSVirtualSpace::expand_by(unsigned long)+0x53
+V  [libjvm.so+0x879f73]  PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x103
+V  [libjvm.so+0x29a593]  AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long)+0x3e3
+V  [libjvm.so+0x83f325]  ParallelScavengeHeap::initialize()+0x4d5
+V  [libjvm.so+0x97295a]  Universe::initialize_heap()+0xca
+V  [libjvm.so+0x973b29]  universe_init()+0x79
+V  [libjvm.so+0x5b0f15]  init_globals()+0x65
+V  [libjvm.so+0x95b47d]  Threads::create_vm(JavaVMInitArgs*, bool*)+0x1ed
+V  [libjvm.so+0x6391c4]  JNI_CreateJavaVM+0x74
+C  [libjli.so+0x2f8e]  JavaMain+0x9e
+
+
+---------------  P R O C E S S  ---------------
+
+Java Threads: ( => current thread )
+
+Other Threads:
+
+=>0x00007f441000a000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=7000, stack(0x00007f44197ca000,0x00007f44198cb000)]
+
+VM state:not at safepoint (not fully initialized)
+
+VM Mutex/Monitor currently owned by a thread: None
+
+GC Heap History (0 events):
+No events
+
+Deoptimization events (0 events):
+No events
+
+Internal exceptions (0 events):
+No events
+
+Events (0 events):
+No events
+
+
+Dynamic libraries:
+00400000-00401000 r-xp 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
+00600000-00601000 rw-p 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
+007be000-007df000 rw-p 00000000 00:00 0                                  [heap]
+75ff80000-77ff80000 rw-p 00000000 00:00 0 
+7d5500000-800000000 rw-p 00000000 00:00 0 
+7f4400000000-7f4400270000 rwxp 00000000 00:00 0 
+7f4400270000-7f4410025000 rw-p 00000000 00:00 0 
+7f4410025000-7f4414000000 ---p 00000000 00:00 0 
+7f4416047000-7f44161b3000 rw-p 00000000 00:00 0 
+7f44161b3000-7f441655d000 rw-p 00000000 00:00 0 
+7f441655d000-7f44166be000 rw-p 00000000 00:00 0 
+7f44166be000-7f4416ab4000 rw-p 00000000 00:00 0 
+7f4416ab4000-7f4416ace000 r-xp 00000000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f4416ace000-7f4416cce000 ---p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f4416cce000-7f4416ccf000 rw-p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f4416ccf000-7f4416cda000 r-xp 00000000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f4416cda000-7f4416ed9000 ---p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f4416ed9000-7f4416eda000 r--p 0000a000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f4416eda000-7f4416edb000 rw-p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f4416edb000-7f4416ee6000 r-xp 00000000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f4416ee6000-7f44170e5000 ---p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f44170e5000-7f44170e6000 r--p 0000a000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f44170e6000-7f44170e7000 rw-p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f44170e7000-7f44170fe000 r-xp 00000000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f44170fe000-7f44172fd000 ---p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f44172fd000-7f44172fe000 r--p 00016000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f44172fe000-7f44172ff000 rw-p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f44172ff000-7f4417301000 rw-p 00000000 00:00 0 
+7f4417301000-7f441730a000 r-xp 00000000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f441730a000-7f4417509000 ---p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f4417509000-7f441750a000 r--p 00008000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f441750a000-7f441750b000 rw-p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f441750b000-7f4417534000 r-xp 00000000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f4417534000-7f4417734000 ---p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f4417734000-7f4417736000 rw-p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f4417736000-7f4417743000 r-xp 00000000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f4417743000-7f4417942000 ---p 0000d000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f4417942000-7f4417944000 rw-p 0000c000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f4417944000-7f441794b000 r-xp 00000000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f441794b000-7f4417b4a000 ---p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f4417b4a000-7f4417b4b000 r--p 00006000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f4417b4b000-7f4417b4c000 rw-p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f4417b4c000-7f4417c51000 r-xp 00000000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f4417c51000-7f4417e50000 ---p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f4417e50000-7f4417e51000 r--p 00104000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f4417e51000-7f4417e52000 rw-p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f4417e52000-7f44189c4000 r-xp 00000000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f44189c4000-7f4418bc3000 ---p 00b72000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f4418bc3000-7f4418c88000 rw-p 00b71000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f4418c88000-7f4418cc8000 rw-p 00000000 00:00 0 
+7f4418cc8000-7f4418e83000 r-xp 00000000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f4418e83000-7f4419083000 ---p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f4419083000-7f4419087000 r--p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f4419087000-7f4419089000 rw-p 001bf000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f4419089000-7f441908e000 rw-p 00000000 00:00 0 
+7f441908e000-7f4419091000 r-xp 00000000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f4419091000-7f4419290000 ---p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f4419290000-7f4419291000 r--p 00002000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f4419291000-7f4419292000 rw-p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f4419292000-7f44192a8000 r-xp 00000000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f44192a8000-7f44194a8000 ---p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f44194a8000-7f44194a9000 rw-p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f44194a9000-7f44194c2000 r-xp 00000000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f44194c2000-7f44196c1000 ---p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f44196c1000-7f44196c2000 r--p 00018000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f44196c2000-7f44196c3000 rw-p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f44196c3000-7f44196c7000 rw-p 00000000 00:00 0 
+7f44196c7000-7f44196ea000 r-xp 00000000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f44196ef000-7f44197ca000 rw-p 00000000 00:00 0 
+7f44197ca000-7f44197cd000 ---p 00000000 00:00 0 
+7f44197cd000-7f44198cf000 rw-p 00000000 00:00 0                          [stack:7000]
+7f44198db000-7f44198dd000 rw-p 00000000 00:00 0 
+7f44198dd000-7f44198e5000 rw-s 00000000 08:04 3539299                    /tmp/hsperfdata_jeanlyn/6969
+7f44198e5000-7f44198e6000 rw-p 00000000 00:00 0 
+7f44198e6000-7f44198e7000 r--p 00000000 00:00 0 
+7f44198e7000-7f44198e9000 rw-p 00000000 00:00 0 
+7f44198e9000-7f44198ea000 r--p 00022000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f44198ea000-7f44198eb000 rw-p 00023000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f44198eb000-7f44198ec000 rw-p 00000000 00:00 0 
+7ffff2cbb000-7ffff2cdd000 rw-p 00000000 00:00 0                          [stack]
+7ffff2ce4000-7ffff2ce6000 r-xp 00000000 00:00 0                          [vdso]
+ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
+
+VM Arguments:
+jvm_args: -Xms2048m -Xmx2048m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=256m 
+java_command: build/sbt-launch-0.13.7.jar scalastyle
+Launcher Type: SUN_STANDARD
+
+Environment Variables:
+JAVA_HOME=/home/jeanlyn/software/jdk1.7.0_71
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/jeanlyn/software/jdk1.7.0_71/bin:/home/jeanlyn/software/apache-maven-3.2.5/bin:/home/jeanlyn/software/idea-IU-139.659.2/bin:/home/jeanlyn/software/hadoop-2.2.0/bin:/home/jeanlyn/software/hadoop-2.2.0/sbin:/home/jeanlyn/software/sublimetext2:/home/jeanlyn/software/scala-2.10.4/bin
+SHELL=/bin/bash
+DISPLAY=:0
+
+Signal Handlers:
+SIGSEGV: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGBUS: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGFPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGPIPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGXFSZ: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGILL: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGUSR1: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGUSR2: [libjvm.so+0x81cb90], sa_mask[0]=0x00000000, sa_flags=0x10000004
+SIGHUP: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGINT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGTERM: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGQUIT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+
+
+---------------  S Y S T E M  ---------------
+
+OS:jessie/sid
+
+uname:Linux 3.13.0-43-generic #72-Ubuntu SMP Mon Dec 8 19:35:06 UTC 2014 x86_64
+libc:glibc 2.19 NPTL 2.19 
+rlimit: STACK 8192k, CORE 0k, NPROC 30300, NOFILE 4096, AS infinity
+load average:0.53 0.59 0.59
+
+/proc/meminfo:
+MemTotal:        3900112 kB
+MemFree:          237560 kB
+Buffers:          178208 kB
+Cached:          1032580 kB
+SwapCached:            0 kB
+Active:          2829092 kB
+Inactive:         563368 kB
+Active(anon):    2183384 kB
+Inactive(anon):    41420 kB
+Active(file):     645708 kB
+Inactive(file):   521948 kB
+Unevictable:         288 kB
+Mlocked:             288 kB
+SwapTotal:             0 kB
+SwapFree:              0 kB
+Dirty:             14792 kB
+Writeback:             0 kB
+AnonPages:       2181888 kB
+Mapped:           247680 kB
+Shmem:             42968 kB
+Slab:             147352 kB
+SReclaimable:     110184 kB
+SUnreclaim:        37168 kB
+KernelStack:        5920 kB
+PageTables:        48544 kB
+NFS_Unstable:          0 kB
+Bounce:                0 kB
+WritebackTmp:          0 kB
+CommitLimit:     1950056 kB
+Committed_AS:    8477676 kB
+VmallocTotal:   34359738367 kB
+VmallocUsed:       91520 kB
+VmallocChunk:   34359633612 kB
+HardwareCorrupted:     0 kB
+AnonHugePages:    876544 kB
+HugePages_Total:       0
+HugePages_Free:        0
+HugePages_Rsvd:        0
+HugePages_Surp:        0
+Hugepagesize:       2048 kB
+DirectMap4k:       59344 kB
+DirectMap2M:     3985408 kB
+
+
+CPU:total 4 (2 cores per cpu, 2 threads per core) family 6 model 37 stepping 5, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, ht, tsc, tscinvbit
+
+/proc/cpuinfo:
+processor	: 0
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1199.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 0
+initial apicid	: 0
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 1
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1199.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 2
+cpu cores	: 2
+apicid		: 4
+initial apicid	: 4
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 2
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 2534.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 1
+initial apicid	: 1
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 3
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1733.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 2
+cpu cores	: 2
+apicid		: 5
+initial apicid	: 5
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+
+
+Memory: 4k page, physical 3900112k(237560k free), swap 0k(0k free)
+
+vm_info: Java HotSpot(TM) 64-Bit Server VM (24.71-b01) for linux-amd64 JRE (1.7.0_71-b14), built on Sep 26 2014 16:41:40 by "java_re" with gcc 4.3.0 20080428 (Red Hat 4.3.0-8)
+
+time: Sun Feb  8 03:10:22 2015
+elapsed time: 0 seconds
+
diff --git a/hs_err_pid7002.log b/hs_err_pid7002.log
new file mode 100644
index 0000000000000..93487290f83d3
--- /dev/null
+++ b/hs_err_pid7002.log
@@ -0,0 +1,352 @@
+#
+# There is insufficient memory for the Java Runtime Environment to continue.
+# Native memory allocation (malloc) failed to allocate 1431830528 bytes for committing reserved memory.
+# Possible reasons:
+#   The system is out of physical RAM or swap space
+#   In 32 bit mode, the process size limit was hit
+# Possible solutions:
+#   Reduce memory load on the system
+#   Increase physical memory or swap space
+#   Check if swap backing store is full
+#   Use 64 bit Java on a 64 bit OS
+#   Decrease Java heap size (-Xmx/-Xms)
+#   Decrease number of Java threads
+#   Decrease Java thread stack sizes (-Xss)
+#   Set larger code cache with -XX:ReservedCodeCacheSize=
+# This output file may be truncated or incomplete.
+#
+#  Out of Memory Error (os_linux.cpp:2745), pid=7002, tid=140013740181248
+#
+# JRE version:  (7.0_71-b14) (build )
+# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.71-b01 mixed mode linux-amd64 compressed oops)
+# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
+#
+
+---------------  T H R E A D  ---------------
+
+Current thread (0x00007f577400a000):  JavaThread "Unknown thread" [_thread_in_vm, id=7013, stack(0x00007f577d2f4000,0x00007f577d3f5000)]
+
+Stack: [0x00007f577d2f4000,0x00007f577d3f5000],  sp=0x00007f577d3f3230,  free space=1020k
+Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
+V  [libjvm.so+0x9a0a1a]  VMError::report_and_die()+0x2ea
+V  [libjvm.so+0x4974bb]  report_vm_out_of_memory(char const*, int, unsigned long, char const*)+0x9b
+V  [libjvm.so+0x81e4de]  os::Linux::commit_memory_impl(char*, unsigned long, bool)+0xfe
+V  [libjvm.so+0x81ea29]  os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x29
+V  [libjvm.so+0x8187da]  os::commit_memory(char*, unsigned long, unsigned long, bool)+0x2a
+V  [libjvm.so+0x88b2e3]  PSVirtualSpace::expand_by(unsigned long)+0x53
+V  [libjvm.so+0x879f73]  PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x103
+V  [libjvm.so+0x29a593]  AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long)+0x3e3
+V  [libjvm.so+0x83f325]  ParallelScavengeHeap::initialize()+0x4d5
+V  [libjvm.so+0x97295a]  Universe::initialize_heap()+0xca
+V  [libjvm.so+0x973b29]  universe_init()+0x79
+V  [libjvm.so+0x5b0f15]  init_globals()+0x65
+V  [libjvm.so+0x95b47d]  Threads::create_vm(JavaVMInitArgs*, bool*)+0x1ed
+V  [libjvm.so+0x6391c4]  JNI_CreateJavaVM+0x74
+C  [libjli.so+0x2f8e]  JavaMain+0x9e
+
+
+---------------  P R O C E S S  ---------------
+
+Java Threads: ( => current thread )
+
+Other Threads:
+
+=>0x00007f577400a000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=7013, stack(0x00007f577d2f4000,0x00007f577d3f5000)]
+
+VM state:not at safepoint (not fully initialized)
+
+VM Mutex/Monitor currently owned by a thread: None
+
+GC Heap History (0 events):
+No events
+
+Deoptimization events (0 events):
+No events
+
+Internal exceptions (0 events):
+No events
+
+Events (0 events):
+No events
+
+
+Dynamic libraries:
+00400000-00401000 r-xp 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
+00600000-00601000 rw-p 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
+0083e000-0085f000 rw-p 00000000 00:00 0                                  [heap]
+75ff80000-77ff80000 rw-p 00000000 00:00 0 
+7d5500000-800000000 rw-p 00000000 00:00 0 
+7f5764000000-7f5764270000 rwxp 00000000 00:00 0 
+7f5764270000-7f5774025000 rw-p 00000000 00:00 0 
+7f5774025000-7f5778000000 ---p 00000000 00:00 0 
+7f5779b71000-7f5779cdd000 rw-p 00000000 00:00 0 
+7f5779cdd000-7f577a087000 rw-p 00000000 00:00 0 
+7f577a087000-7f577a1e8000 rw-p 00000000 00:00 0 
+7f577a1e8000-7f577a5de000 rw-p 00000000 00:00 0 
+7f577a5de000-7f577a5f8000 r-xp 00000000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f577a5f8000-7f577a7f8000 ---p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f577a7f8000-7f577a7f9000 rw-p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
+7f577a7f9000-7f577a804000 r-xp 00000000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f577a804000-7f577aa03000 ---p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f577aa03000-7f577aa04000 r--p 0000a000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f577aa04000-7f577aa05000 rw-p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
+7f577aa05000-7f577aa10000 r-xp 00000000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f577aa10000-7f577ac0f000 ---p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f577ac0f000-7f577ac10000 r--p 0000a000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f577ac10000-7f577ac11000 rw-p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
+7f577ac11000-7f577ac28000 r-xp 00000000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f577ac28000-7f577ae27000 ---p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f577ae27000-7f577ae28000 r--p 00016000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f577ae28000-7f577ae29000 rw-p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
+7f577ae29000-7f577ae2b000 rw-p 00000000 00:00 0 
+7f577ae2b000-7f577ae34000 r-xp 00000000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f577ae34000-7f577b033000 ---p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f577b033000-7f577b034000 r--p 00008000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f577b034000-7f577b035000 rw-p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
+7f577b035000-7f577b05e000 r-xp 00000000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f577b05e000-7f577b25e000 ---p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f577b25e000-7f577b260000 rw-p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
+7f577b260000-7f577b26d000 r-xp 00000000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f577b26d000-7f577b46c000 ---p 0000d000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f577b46c000-7f577b46e000 rw-p 0000c000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
+7f577b46e000-7f577b475000 r-xp 00000000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f577b475000-7f577b674000 ---p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f577b674000-7f577b675000 r--p 00006000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f577b675000-7f577b676000 rw-p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
+7f577b676000-7f577b77b000 r-xp 00000000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f577b77b000-7f577b97a000 ---p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f577b97a000-7f577b97b000 r--p 00104000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f577b97b000-7f577b97c000 rw-p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
+7f577b97c000-7f577c4ee000 r-xp 00000000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f577c4ee000-7f577c6ed000 ---p 00b72000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f577c6ed000-7f577c7b2000 rw-p 00b71000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
+7f577c7b2000-7f577c7f2000 rw-p 00000000 00:00 0 
+7f577c7f2000-7f577c9ad000 r-xp 00000000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f577c9ad000-7f577cbad000 ---p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f577cbad000-7f577cbb1000 r--p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f577cbb1000-7f577cbb3000 rw-p 001bf000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
+7f577cbb3000-7f577cbb8000 rw-p 00000000 00:00 0 
+7f577cbb8000-7f577cbbb000 r-xp 00000000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f577cbbb000-7f577cdba000 ---p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f577cdba000-7f577cdbb000 r--p 00002000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f577cdbb000-7f577cdbc000 rw-p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
+7f577cdbc000-7f577cdd2000 r-xp 00000000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f577cdd2000-7f577cfd2000 ---p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f577cfd2000-7f577cfd3000 rw-p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
+7f577cfd3000-7f577cfec000 r-xp 00000000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f577cfec000-7f577d1eb000 ---p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f577d1eb000-7f577d1ec000 r--p 00018000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f577d1ec000-7f577d1ed000 rw-p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
+7f577d1ed000-7f577d1f1000 rw-p 00000000 00:00 0 
+7f577d1f1000-7f577d214000 r-xp 00000000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f577d219000-7f577d2f4000 rw-p 00000000 00:00 0 
+7f577d2f4000-7f577d2f7000 ---p 00000000 00:00 0 
+7f577d2f7000-7f577d3f9000 rw-p 00000000 00:00 0                          [stack:7013]
+7f577d405000-7f577d407000 rw-p 00000000 00:00 0 
+7f577d407000-7f577d40f000 rw-s 00000000 08:04 3539299                    /tmp/hsperfdata_jeanlyn/7002
+7f577d40f000-7f577d410000 rw-p 00000000 00:00 0 
+7f577d410000-7f577d411000 r--p 00000000 00:00 0 
+7f577d411000-7f577d413000 rw-p 00000000 00:00 0 
+7f577d413000-7f577d414000 r--p 00022000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f577d414000-7f577d415000 rw-p 00023000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
+7f577d415000-7f577d416000 rw-p 00000000 00:00 0 
+7fff34d22000-7fff34d44000 rw-p 00000000 00:00 0                          [stack]
+7fff34d59000-7fff34d5b000 r-xp 00000000 00:00 0                          [vdso]
+ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
+
+VM Arguments:
+jvm_args: -Xms2048m -Xmx2048m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=256m -Dhadoop.version=2.2.0 
+java_command: build/sbt-launch-0.13.7.jar scalastyle
+Launcher Type: SUN_STANDARD
+
+Environment Variables:
+JAVA_HOME=/home/jeanlyn/software/jdk1.7.0_71
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/jeanlyn/software/jdk1.7.0_71/bin:/home/jeanlyn/software/apache-maven-3.2.5/bin:/home/jeanlyn/software/idea-IU-139.659.2/bin:/home/jeanlyn/software/hadoop-2.2.0/bin:/home/jeanlyn/software/hadoop-2.2.0/sbin:/home/jeanlyn/software/sublimetext2:/home/jeanlyn/software/scala-2.10.4/bin
+SHELL=/bin/bash
+DISPLAY=:0
+
+Signal Handlers:
+SIGSEGV: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGBUS: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGFPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGPIPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGXFSZ: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGILL: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
+SIGUSR1: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGUSR2: [libjvm.so+0x81cb90], sa_mask[0]=0x00000000, sa_flags=0x10000004
+SIGHUP: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGINT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGTERM: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+SIGQUIT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
+
+
+---------------  S Y S T E M  ---------------
+
+OS:jessie/sid
+
+uname:Linux 3.13.0-43-generic #72-Ubuntu SMP Mon Dec 8 19:35:06 UTC 2014 x86_64
+libc:glibc 2.19 NPTL 2.19 
+rlimit: STACK 8192k, CORE 0k, NPROC 30300, NOFILE 4096, AS infinity
+load average:0.53 0.59 0.59
+
+/proc/meminfo:
+MemTotal:        3900112 kB
+MemFree:          236544 kB
+Buffers:          178208 kB
+Cached:          1032800 kB
+SwapCached:            0 kB
+Active:          2828940 kB
+Inactive:         563516 kB
+Active(anon):    2183232 kB
+Inactive(anon):    41420 kB
+Active(file):     645708 kB
+Inactive(file):   522096 kB
+Unevictable:         288 kB
+Mlocked:             288 kB
+SwapTotal:             0 kB
+SwapFree:              0 kB
+Dirty:             14792 kB
+Writeback:             0 kB
+AnonPages:       2181848 kB
+Mapped:           247900 kB
+Shmem:             42968 kB
+Slab:             147352 kB
+SReclaimable:     110184 kB
+SUnreclaim:        37168 kB
+KernelStack:        5920 kB
+PageTables:        48764 kB
+NFS_Unstable:          0 kB
+Bounce:                0 kB
+WritebackTmp:          0 kB
+CommitLimit:     1950056 kB
+Committed_AS:    8476656 kB
+VmallocTotal:   34359738367 kB
+VmallocUsed:       91520 kB
+VmallocChunk:   34359633612 kB
+HardwareCorrupted:     0 kB
+AnonHugePages:    876544 kB
+HugePages_Total:       0
+HugePages_Free:        0
+HugePages_Rsvd:        0
+HugePages_Surp:        0
+Hugepagesize:       2048 kB
+DirectMap4k:       59344 kB
+DirectMap2M:     3985408 kB
+
+
+CPU:total 4 (2 cores per cpu, 2 threads per core) family 6 model 37 stepping 5, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, ht, tsc, tscinvbit
+
+/proc/cpuinfo:
+processor	: 0
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1199.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 0
+initial apicid	: 0
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 1
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1199.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 2
+cpu cores	: 2
+apicid		: 4
+initial apicid	: 4
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 2
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1199.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 0
+cpu cores	: 2
+apicid		: 1
+initial apicid	: 1
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+processor	: 3
+vendor_id	: GenuineIntel
+cpu family	: 6
+model		: 37
+model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
+stepping	: 5
+microcode	: 0x2
+cpu MHz		: 1599.000
+cache size	: 3072 KB
+physical id	: 0
+siblings	: 4
+core id		: 2
+cpu cores	: 2
+apicid		: 5
+initial apicid	: 5
+fpu		: yes
+fpu_exception	: yes
+cpuid level	: 11
+wp		: yes
+flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
+bogomips	: 5053.83
+clflush size	: 64
+cache_alignment	: 64
+address sizes	: 36 bits physical, 48 bits virtual
+power management:
+
+
+
+Memory: 4k page, physical 3900112k(236544k free), swap 0k(0k free)
+
+vm_info: Java HotSpot(TM) 64-Bit Server VM (24.71-b01) for linux-amd64 JRE (1.7.0_71-b14), built on Sep 26 2014 16:41:40 by "java_re" with gcc 4.3.0 20080428 (Red Hat 4.3.0-8)
+
+time: Sun Feb  8 03:10:22 2015
+elapsed time: 0 seconds
+
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index d618ec5298526..43991a99d6403 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -280,7 +280,8 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       tableDeser: Deserializer): Iterator[Row] = {
 
     val soi = HiveShim.getConvertedOI(
-      rawDeser.getObjectInspector, tableDeser.getObjectInspector).asInstanceOf[StructObjectInspector]
+      rawDeser.getObjectInspector,
+      tableDeser.getObjectInspector).asInstanceOf[StructObjectInspector]
 
     val inputFields = soi.getAllStructFieldRefs
 
@@ -288,8 +289,10 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       (inputFields.get(ordinal), ordinal)
     }.unzip
 
-    // Builds specific unwrappers ahead of time according to object inspector types to avoid pattern
-    // matching and branching costs per row.
+    /**
+     * Builds specific unwrappers ahead of time according to object inspector
+     * types to avoid pattern matching and branching costs per row.
+     */
     val unwrappers: Seq[(Any, MutableRow, Int) => Unit] = fieldRefs.map {
       _.getFieldObjectInspector match {
         case oi: BooleanObjectInspector =>

From 1e8b30cea76d1f0594beb37a47252118123b649d Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Sun, 8 Feb 2015 03:15:55 +0800
Subject: [PATCH 10/13] fix code style

---
 hs_err_pid6969.log | 352 ---------------------------------------------
 hs_err_pid7002.log | 352 ---------------------------------------------
 2 files changed, 704 deletions(-)
 delete mode 100644 hs_err_pid6969.log
 delete mode 100644 hs_err_pid7002.log

diff --git a/hs_err_pid6969.log b/hs_err_pid6969.log
deleted file mode 100644
index 176ac9a4af8be..0000000000000
--- a/hs_err_pid6969.log
+++ /dev/null
@@ -1,352 +0,0 @@
-#
-# There is insufficient memory for the Java Runtime Environment to continue.
-# Native memory allocation (malloc) failed to allocate 1431830528 bytes for committing reserved memory.
-# Possible reasons:
-#   The system is out of physical RAM or swap space
-#   In 32 bit mode, the process size limit was hit
-# Possible solutions:
-#   Reduce memory load on the system
-#   Increase physical memory or swap space
-#   Check if swap backing store is full
-#   Use 64 bit Java on a 64 bit OS
-#   Decrease Java heap size (-Xmx/-Xms)
-#   Decrease number of Java threads
-#   Decrease Java thread stack sizes (-Xss)
-#   Set larger code cache with -XX:ReservedCodeCacheSize=
-# This output file may be truncated or incomplete.
-#
-#  Out of Memory Error (os_linux.cpp:2745), pid=6969, tid=139930463151872
-#
-# JRE version:  (7.0_71-b14) (build )
-# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.71-b01 mixed mode linux-amd64 compressed oops)
-# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
-#
-
----------------  T H R E A D  ---------------
-
-Current thread (0x00007f441000a000):  JavaThread "Unknown thread" [_thread_in_vm, id=7000, stack(0x00007f44197ca000,0x00007f44198cb000)]
-
-Stack: [0x00007f44197ca000,0x00007f44198cb000],  sp=0x00007f44198c9230,  free space=1020k
-Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
-V  [libjvm.so+0x9a0a1a]  VMError::report_and_die()+0x2ea
-V  [libjvm.so+0x4974bb]  report_vm_out_of_memory(char const*, int, unsigned long, char const*)+0x9b
-V  [libjvm.so+0x81e4de]  os::Linux::commit_memory_impl(char*, unsigned long, bool)+0xfe
-V  [libjvm.so+0x81ea29]  os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x29
-V  [libjvm.so+0x8187da]  os::commit_memory(char*, unsigned long, unsigned long, bool)+0x2a
-V  [libjvm.so+0x88b2e3]  PSVirtualSpace::expand_by(unsigned long)+0x53
-V  [libjvm.so+0x879f73]  PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x103
-V  [libjvm.so+0x29a593]  AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long)+0x3e3
-V  [libjvm.so+0x83f325]  ParallelScavengeHeap::initialize()+0x4d5
-V  [libjvm.so+0x97295a]  Universe::initialize_heap()+0xca
-V  [libjvm.so+0x973b29]  universe_init()+0x79
-V  [libjvm.so+0x5b0f15]  init_globals()+0x65
-V  [libjvm.so+0x95b47d]  Threads::create_vm(JavaVMInitArgs*, bool*)+0x1ed
-V  [libjvm.so+0x6391c4]  JNI_CreateJavaVM+0x74
-C  [libjli.so+0x2f8e]  JavaMain+0x9e
-
-
----------------  P R O C E S S  ---------------
-
-Java Threads: ( => current thread )
-
-Other Threads:
-
-=>0x00007f441000a000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=7000, stack(0x00007f44197ca000,0x00007f44198cb000)]
-
-VM state:not at safepoint (not fully initialized)
-
-VM Mutex/Monitor currently owned by a thread: None
-
-GC Heap History (0 events):
-No events
-
-Deoptimization events (0 events):
-No events
-
-Internal exceptions (0 events):
-No events
-
-Events (0 events):
-No events
-
-
-Dynamic libraries:
-00400000-00401000 r-xp 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
-00600000-00601000 rw-p 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
-007be000-007df000 rw-p 00000000 00:00 0                                  [heap]
-75ff80000-77ff80000 rw-p 00000000 00:00 0 
-7d5500000-800000000 rw-p 00000000 00:00 0 
-7f4400000000-7f4400270000 rwxp 00000000 00:00 0 
-7f4400270000-7f4410025000 rw-p 00000000 00:00 0 
-7f4410025000-7f4414000000 ---p 00000000 00:00 0 
-7f4416047000-7f44161b3000 rw-p 00000000 00:00 0 
-7f44161b3000-7f441655d000 rw-p 00000000 00:00 0 
-7f441655d000-7f44166be000 rw-p 00000000 00:00 0 
-7f44166be000-7f4416ab4000 rw-p 00000000 00:00 0 
-7f4416ab4000-7f4416ace000 r-xp 00000000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f4416ace000-7f4416cce000 ---p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f4416cce000-7f4416ccf000 rw-p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f4416ccf000-7f4416cda000 r-xp 00000000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f4416cda000-7f4416ed9000 ---p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f4416ed9000-7f4416eda000 r--p 0000a000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f4416eda000-7f4416edb000 rw-p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f4416edb000-7f4416ee6000 r-xp 00000000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f4416ee6000-7f44170e5000 ---p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f44170e5000-7f44170e6000 r--p 0000a000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f44170e6000-7f44170e7000 rw-p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f44170e7000-7f44170fe000 r-xp 00000000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f44170fe000-7f44172fd000 ---p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f44172fd000-7f44172fe000 r--p 00016000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f44172fe000-7f44172ff000 rw-p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f44172ff000-7f4417301000 rw-p 00000000 00:00 0 
-7f4417301000-7f441730a000 r-xp 00000000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f441730a000-7f4417509000 ---p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f4417509000-7f441750a000 r--p 00008000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f441750a000-7f441750b000 rw-p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f441750b000-7f4417534000 r-xp 00000000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f4417534000-7f4417734000 ---p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f4417734000-7f4417736000 rw-p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f4417736000-7f4417743000 r-xp 00000000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f4417743000-7f4417942000 ---p 0000d000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f4417942000-7f4417944000 rw-p 0000c000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f4417944000-7f441794b000 r-xp 00000000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f441794b000-7f4417b4a000 ---p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f4417b4a000-7f4417b4b000 r--p 00006000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f4417b4b000-7f4417b4c000 rw-p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f4417b4c000-7f4417c51000 r-xp 00000000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f4417c51000-7f4417e50000 ---p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f4417e50000-7f4417e51000 r--p 00104000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f4417e51000-7f4417e52000 rw-p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f4417e52000-7f44189c4000 r-xp 00000000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f44189c4000-7f4418bc3000 ---p 00b72000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f4418bc3000-7f4418c88000 rw-p 00b71000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f4418c88000-7f4418cc8000 rw-p 00000000 00:00 0 
-7f4418cc8000-7f4418e83000 r-xp 00000000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f4418e83000-7f4419083000 ---p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f4419083000-7f4419087000 r--p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f4419087000-7f4419089000 rw-p 001bf000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f4419089000-7f441908e000 rw-p 00000000 00:00 0 
-7f441908e000-7f4419091000 r-xp 00000000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f4419091000-7f4419290000 ---p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f4419290000-7f4419291000 r--p 00002000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f4419291000-7f4419292000 rw-p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f4419292000-7f44192a8000 r-xp 00000000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f44192a8000-7f44194a8000 ---p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f44194a8000-7f44194a9000 rw-p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f44194a9000-7f44194c2000 r-xp 00000000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f44194c2000-7f44196c1000 ---p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f44196c1000-7f44196c2000 r--p 00018000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f44196c2000-7f44196c3000 rw-p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f44196c3000-7f44196c7000 rw-p 00000000 00:00 0 
-7f44196c7000-7f44196ea000 r-xp 00000000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f44196ef000-7f44197ca000 rw-p 00000000 00:00 0 
-7f44197ca000-7f44197cd000 ---p 00000000 00:00 0 
-7f44197cd000-7f44198cf000 rw-p 00000000 00:00 0                          [stack:7000]
-7f44198db000-7f44198dd000 rw-p 00000000 00:00 0 
-7f44198dd000-7f44198e5000 rw-s 00000000 08:04 3539299                    /tmp/hsperfdata_jeanlyn/6969
-7f44198e5000-7f44198e6000 rw-p 00000000 00:00 0 
-7f44198e6000-7f44198e7000 r--p 00000000 00:00 0 
-7f44198e7000-7f44198e9000 rw-p 00000000 00:00 0 
-7f44198e9000-7f44198ea000 r--p 00022000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f44198ea000-7f44198eb000 rw-p 00023000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f44198eb000-7f44198ec000 rw-p 00000000 00:00 0 
-7ffff2cbb000-7ffff2cdd000 rw-p 00000000 00:00 0                          [stack]
-7ffff2ce4000-7ffff2ce6000 r-xp 00000000 00:00 0                          [vdso]
-ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
-
-VM Arguments:
-jvm_args: -Xms2048m -Xmx2048m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=256m 
-java_command: build/sbt-launch-0.13.7.jar scalastyle
-Launcher Type: SUN_STANDARD
-
-Environment Variables:
-JAVA_HOME=/home/jeanlyn/software/jdk1.7.0_71
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/jeanlyn/software/jdk1.7.0_71/bin:/home/jeanlyn/software/apache-maven-3.2.5/bin:/home/jeanlyn/software/idea-IU-139.659.2/bin:/home/jeanlyn/software/hadoop-2.2.0/bin:/home/jeanlyn/software/hadoop-2.2.0/sbin:/home/jeanlyn/software/sublimetext2:/home/jeanlyn/software/scala-2.10.4/bin
-SHELL=/bin/bash
-DISPLAY=:0
-
-Signal Handlers:
-SIGSEGV: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGBUS: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGFPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGPIPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGXFSZ: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGILL: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGUSR1: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGUSR2: [libjvm.so+0x81cb90], sa_mask[0]=0x00000000, sa_flags=0x10000004
-SIGHUP: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGINT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGTERM: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGQUIT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-
-
----------------  S Y S T E M  ---------------
-
-OS:jessie/sid
-
-uname:Linux 3.13.0-43-generic #72-Ubuntu SMP Mon Dec 8 19:35:06 UTC 2014 x86_64
-libc:glibc 2.19 NPTL 2.19 
-rlimit: STACK 8192k, CORE 0k, NPROC 30300, NOFILE 4096, AS infinity
-load average:0.53 0.59 0.59
-
-/proc/meminfo:
-MemTotal:        3900112 kB
-MemFree:          237560 kB
-Buffers:          178208 kB
-Cached:          1032580 kB
-SwapCached:            0 kB
-Active:          2829092 kB
-Inactive:         563368 kB
-Active(anon):    2183384 kB
-Inactive(anon):    41420 kB
-Active(file):     645708 kB
-Inactive(file):   521948 kB
-Unevictable:         288 kB
-Mlocked:             288 kB
-SwapTotal:             0 kB
-SwapFree:              0 kB
-Dirty:             14792 kB
-Writeback:             0 kB
-AnonPages:       2181888 kB
-Mapped:           247680 kB
-Shmem:             42968 kB
-Slab:             147352 kB
-SReclaimable:     110184 kB
-SUnreclaim:        37168 kB
-KernelStack:        5920 kB
-PageTables:        48544 kB
-NFS_Unstable:          0 kB
-Bounce:                0 kB
-WritebackTmp:          0 kB
-CommitLimit:     1950056 kB
-Committed_AS:    8477676 kB
-VmallocTotal:   34359738367 kB
-VmallocUsed:       91520 kB
-VmallocChunk:   34359633612 kB
-HardwareCorrupted:     0 kB
-AnonHugePages:    876544 kB
-HugePages_Total:       0
-HugePages_Free:        0
-HugePages_Rsvd:        0
-HugePages_Surp:        0
-Hugepagesize:       2048 kB
-DirectMap4k:       59344 kB
-DirectMap2M:     3985408 kB
-
-
-CPU:total 4 (2 cores per cpu, 2 threads per core) family 6 model 37 stepping 5, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, ht, tsc, tscinvbit
-
-/proc/cpuinfo:
-processor	: 0
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1199.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 0
-cpu cores	: 2
-apicid		: 0
-initial apicid	: 0
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 1
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1199.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 2
-cpu cores	: 2
-apicid		: 4
-initial apicid	: 4
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 2
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 2534.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 0
-cpu cores	: 2
-apicid		: 1
-initial apicid	: 1
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 3
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1733.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 2
-cpu cores	: 2
-apicid		: 5
-initial apicid	: 5
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-
-
-Memory: 4k page, physical 3900112k(237560k free), swap 0k(0k free)
-
-vm_info: Java HotSpot(TM) 64-Bit Server VM (24.71-b01) for linux-amd64 JRE (1.7.0_71-b14), built on Sep 26 2014 16:41:40 by "java_re" with gcc 4.3.0 20080428 (Red Hat 4.3.0-8)
-
-time: Sun Feb  8 03:10:22 2015
-elapsed time: 0 seconds
-
diff --git a/hs_err_pid7002.log b/hs_err_pid7002.log
deleted file mode 100644
index 93487290f83d3..0000000000000
--- a/hs_err_pid7002.log
+++ /dev/null
@@ -1,352 +0,0 @@
-#
-# There is insufficient memory for the Java Runtime Environment to continue.
-# Native memory allocation (malloc) failed to allocate 1431830528 bytes for committing reserved memory.
-# Possible reasons:
-#   The system is out of physical RAM or swap space
-#   In 32 bit mode, the process size limit was hit
-# Possible solutions:
-#   Reduce memory load on the system
-#   Increase physical memory or swap space
-#   Check if swap backing store is full
-#   Use 64 bit Java on a 64 bit OS
-#   Decrease Java heap size (-Xmx/-Xms)
-#   Decrease number of Java threads
-#   Decrease Java thread stack sizes (-Xss)
-#   Set larger code cache with -XX:ReservedCodeCacheSize=
-# This output file may be truncated or incomplete.
-#
-#  Out of Memory Error (os_linux.cpp:2745), pid=7002, tid=140013740181248
-#
-# JRE version:  (7.0_71-b14) (build )
-# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.71-b01 mixed mode linux-amd64 compressed oops)
-# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
-#
-
----------------  T H R E A D  ---------------
-
-Current thread (0x00007f577400a000):  JavaThread "Unknown thread" [_thread_in_vm, id=7013, stack(0x00007f577d2f4000,0x00007f577d3f5000)]
-
-Stack: [0x00007f577d2f4000,0x00007f577d3f5000],  sp=0x00007f577d3f3230,  free space=1020k
-Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
-V  [libjvm.so+0x9a0a1a]  VMError::report_and_die()+0x2ea
-V  [libjvm.so+0x4974bb]  report_vm_out_of_memory(char const*, int, unsigned long, char const*)+0x9b
-V  [libjvm.so+0x81e4de]  os::Linux::commit_memory_impl(char*, unsigned long, bool)+0xfe
-V  [libjvm.so+0x81ea29]  os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x29
-V  [libjvm.so+0x8187da]  os::commit_memory(char*, unsigned long, unsigned long, bool)+0x2a
-V  [libjvm.so+0x88b2e3]  PSVirtualSpace::expand_by(unsigned long)+0x53
-V  [libjvm.so+0x879f73]  PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x103
-V  [libjvm.so+0x29a593]  AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long)+0x3e3
-V  [libjvm.so+0x83f325]  ParallelScavengeHeap::initialize()+0x4d5
-V  [libjvm.so+0x97295a]  Universe::initialize_heap()+0xca
-V  [libjvm.so+0x973b29]  universe_init()+0x79
-V  [libjvm.so+0x5b0f15]  init_globals()+0x65
-V  [libjvm.so+0x95b47d]  Threads::create_vm(JavaVMInitArgs*, bool*)+0x1ed
-V  [libjvm.so+0x6391c4]  JNI_CreateJavaVM+0x74
-C  [libjli.so+0x2f8e]  JavaMain+0x9e
-
-
----------------  P R O C E S S  ---------------
-
-Java Threads: ( => current thread )
-
-Other Threads:
-
-=>0x00007f577400a000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=7013, stack(0x00007f577d2f4000,0x00007f577d3f5000)]
-
-VM state:not at safepoint (not fully initialized)
-
-VM Mutex/Monitor currently owned by a thread: None
-
-GC Heap History (0 events):
-No events
-
-Deoptimization events (0 events):
-No events
-
-Internal exceptions (0 events):
-No events
-
-Events (0 events):
-No events
-
-
-Dynamic libraries:
-00400000-00401000 r-xp 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
-00600000-00601000 rw-p 00000000 08:04 265813                             /home/jeanlyn/software/jdk1.7.0_71/bin/java
-0083e000-0085f000 rw-p 00000000 00:00 0                                  [heap]
-75ff80000-77ff80000 rw-p 00000000 00:00 0 
-7d5500000-800000000 rw-p 00000000 00:00 0 
-7f5764000000-7f5764270000 rwxp 00000000 00:00 0 
-7f5764270000-7f5774025000 rw-p 00000000 00:00 0 
-7f5774025000-7f5778000000 ---p 00000000 00:00 0 
-7f5779b71000-7f5779cdd000 rw-p 00000000 00:00 0 
-7f5779cdd000-7f577a087000 rw-p 00000000 00:00 0 
-7f577a087000-7f577a1e8000 rw-p 00000000 00:00 0 
-7f577a1e8000-7f577a5de000 rw-p 00000000 00:00 0 
-7f577a5de000-7f577a5f8000 r-xp 00000000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f577a5f8000-7f577a7f8000 ---p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f577a7f8000-7f577a7f9000 rw-p 0001a000 08:04 919620                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libzip.so
-7f577a7f9000-7f577a804000 r-xp 00000000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f577a804000-7f577aa03000 ---p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f577aa03000-7f577aa04000 r--p 0000a000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f577aa04000-7f577aa05000 rw-p 0000b000 08:04 5511440                    /lib/x86_64-linux-gnu/libnss_files-2.19.so
-7f577aa05000-7f577aa10000 r-xp 00000000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f577aa10000-7f577ac0f000 ---p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f577ac0f000-7f577ac10000 r--p 0000a000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f577ac10000-7f577ac11000 rw-p 0000b000 08:04 5511437                    /lib/x86_64-linux-gnu/libnss_nis-2.19.so
-7f577ac11000-7f577ac28000 r-xp 00000000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f577ac28000-7f577ae27000 ---p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f577ae27000-7f577ae28000 r--p 00016000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f577ae28000-7f577ae29000 rw-p 00017000 08:04 5511431                    /lib/x86_64-linux-gnu/libnsl-2.19.so
-7f577ae29000-7f577ae2b000 rw-p 00000000 00:00 0 
-7f577ae2b000-7f577ae34000 r-xp 00000000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f577ae34000-7f577b033000 ---p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f577b033000-7f577b034000 r--p 00008000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f577b034000-7f577b035000 rw-p 00009000 08:04 5511428                    /lib/x86_64-linux-gnu/libnss_compat-2.19.so
-7f577b035000-7f577b05e000 r-xp 00000000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f577b05e000-7f577b25e000 ---p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f577b25e000-7f577b260000 rw-p 00029000 08:04 919603                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libjava.so
-7f577b260000-7f577b26d000 r-xp 00000000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f577b26d000-7f577b46c000 ---p 0000d000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f577b46c000-7f577b46e000 rw-p 0000c000 08:04 919565                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/libverify.so
-7f577b46e000-7f577b475000 r-xp 00000000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f577b475000-7f577b674000 ---p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f577b674000-7f577b675000 r--p 00006000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f577b675000-7f577b676000 rw-p 00007000 08:04 5511423                    /lib/x86_64-linux-gnu/librt-2.19.so
-7f577b676000-7f577b77b000 r-xp 00000000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f577b77b000-7f577b97a000 ---p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f577b97a000-7f577b97b000 r--p 00104000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f577b97b000-7f577b97c000 rw-p 00105000 08:04 5511430                    /lib/x86_64-linux-gnu/libm-2.19.so
-7f577b97c000-7f577c4ee000 r-xp 00000000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f577c4ee000-7f577c6ed000 ---p 00b72000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f577c6ed000-7f577c7b2000 rw-p 00b71000 08:04 919579                     /home/jeanlyn/software/jdk1.7.0_71/jre/lib/amd64/server/libjvm.so
-7f577c7b2000-7f577c7f2000 rw-p 00000000 00:00 0 
-7f577c7f2000-7f577c9ad000 r-xp 00000000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f577c9ad000-7f577cbad000 ---p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f577cbad000-7f577cbb1000 r--p 001bb000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f577cbb1000-7f577cbb3000 rw-p 001bf000 08:04 5511422                    /lib/x86_64-linux-gnu/libc-2.19.so
-7f577cbb3000-7f577cbb8000 rw-p 00000000 00:00 0 
-7f577cbb8000-7f577cbbb000 r-xp 00000000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f577cbbb000-7f577cdba000 ---p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f577cdba000-7f577cdbb000 r--p 00002000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f577cdbb000-7f577cdbc000 rw-p 00003000 08:04 5511426                    /lib/x86_64-linux-gnu/libdl-2.19.so
-7f577cdbc000-7f577cdd2000 r-xp 00000000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f577cdd2000-7f577cfd2000 ---p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f577cfd2000-7f577cfd3000 rw-p 00016000 08:04 265455                     /home/jeanlyn/software/jdk1.7.0_71/lib/amd64/jli/libjli.so
-7f577cfd3000-7f577cfec000 r-xp 00000000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f577cfec000-7f577d1eb000 ---p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f577d1eb000-7f577d1ec000 r--p 00018000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f577d1ec000-7f577d1ed000 rw-p 00019000 08:04 5511439                    /lib/x86_64-linux-gnu/libpthread-2.19.so
-7f577d1ed000-7f577d1f1000 rw-p 00000000 00:00 0 
-7f577d1f1000-7f577d214000 r-xp 00000000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f577d219000-7f577d2f4000 rw-p 00000000 00:00 0 
-7f577d2f4000-7f577d2f7000 ---p 00000000 00:00 0 
-7f577d2f7000-7f577d3f9000 rw-p 00000000 00:00 0                          [stack:7013]
-7f577d405000-7f577d407000 rw-p 00000000 00:00 0 
-7f577d407000-7f577d40f000 rw-s 00000000 08:04 3539299                    /tmp/hsperfdata_jeanlyn/7002
-7f577d40f000-7f577d410000 rw-p 00000000 00:00 0 
-7f577d410000-7f577d411000 r--p 00000000 00:00 0 
-7f577d411000-7f577d413000 rw-p 00000000 00:00 0 
-7f577d413000-7f577d414000 r--p 00022000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f577d414000-7f577d415000 rw-p 00023000 08:04 5511420                    /lib/x86_64-linux-gnu/ld-2.19.so
-7f577d415000-7f577d416000 rw-p 00000000 00:00 0 
-7fff34d22000-7fff34d44000 rw-p 00000000 00:00 0                          [stack]
-7fff34d59000-7fff34d5b000 r-xp 00000000 00:00 0                          [vdso]
-ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0                  [vsyscall]
-
-VM Arguments:
-jvm_args: -Xms2048m -Xmx2048m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=256m -Dhadoop.version=2.2.0 
-java_command: build/sbt-launch-0.13.7.jar scalastyle
-Launcher Type: SUN_STANDARD
-
-Environment Variables:
-JAVA_HOME=/home/jeanlyn/software/jdk1.7.0_71
-PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/jeanlyn/software/jdk1.7.0_71/bin:/home/jeanlyn/software/apache-maven-3.2.5/bin:/home/jeanlyn/software/idea-IU-139.659.2/bin:/home/jeanlyn/software/hadoop-2.2.0/bin:/home/jeanlyn/software/hadoop-2.2.0/sbin:/home/jeanlyn/software/sublimetext2:/home/jeanlyn/software/scala-2.10.4/bin
-SHELL=/bin/bash
-DISPLAY=:0
-
-Signal Handlers:
-SIGSEGV: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGBUS: [libjvm.so+0x9a1330], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGFPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGPIPE: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGXFSZ: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGILL: [libjvm.so+0x81b390], sa_mask[0]=0x7ffbfeff, sa_flags=0x10000004
-SIGUSR1: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGUSR2: [libjvm.so+0x81cb90], sa_mask[0]=0x00000000, sa_flags=0x10000004
-SIGHUP: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGINT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGTERM: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-SIGQUIT: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
-
-
----------------  S Y S T E M  ---------------
-
-OS:jessie/sid
-
-uname:Linux 3.13.0-43-generic #72-Ubuntu SMP Mon Dec 8 19:35:06 UTC 2014 x86_64
-libc:glibc 2.19 NPTL 2.19 
-rlimit: STACK 8192k, CORE 0k, NPROC 30300, NOFILE 4096, AS infinity
-load average:0.53 0.59 0.59
-
-/proc/meminfo:
-MemTotal:        3900112 kB
-MemFree:          236544 kB
-Buffers:          178208 kB
-Cached:          1032800 kB
-SwapCached:            0 kB
-Active:          2828940 kB
-Inactive:         563516 kB
-Active(anon):    2183232 kB
-Inactive(anon):    41420 kB
-Active(file):     645708 kB
-Inactive(file):   522096 kB
-Unevictable:         288 kB
-Mlocked:             288 kB
-SwapTotal:             0 kB
-SwapFree:              0 kB
-Dirty:             14792 kB
-Writeback:             0 kB
-AnonPages:       2181848 kB
-Mapped:           247900 kB
-Shmem:             42968 kB
-Slab:             147352 kB
-SReclaimable:     110184 kB
-SUnreclaim:        37168 kB
-KernelStack:        5920 kB
-PageTables:        48764 kB
-NFS_Unstable:          0 kB
-Bounce:                0 kB
-WritebackTmp:          0 kB
-CommitLimit:     1950056 kB
-Committed_AS:    8476656 kB
-VmallocTotal:   34359738367 kB
-VmallocUsed:       91520 kB
-VmallocChunk:   34359633612 kB
-HardwareCorrupted:     0 kB
-AnonHugePages:    876544 kB
-HugePages_Total:       0
-HugePages_Free:        0
-HugePages_Rsvd:        0
-HugePages_Surp:        0
-Hugepagesize:       2048 kB
-DirectMap4k:       59344 kB
-DirectMap2M:     3985408 kB
-
-
-CPU:total 4 (2 cores per cpu, 2 threads per core) family 6 model 37 stepping 5, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, ht, tsc, tscinvbit
-
-/proc/cpuinfo:
-processor	: 0
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1199.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 0
-cpu cores	: 2
-apicid		: 0
-initial apicid	: 0
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 1
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1199.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 2
-cpu cores	: 2
-apicid		: 4
-initial apicid	: 4
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 2
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1199.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 0
-cpu cores	: 2
-apicid		: 1
-initial apicid	: 1
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-processor	: 3
-vendor_id	: GenuineIntel
-cpu family	: 6
-model		: 37
-model name	: Intel(R) Core(TM) i5 CPU       M 460  @ 2.53GHz
-stepping	: 5
-microcode	: 0x2
-cpu MHz		: 1599.000
-cache size	: 3072 KB
-physical id	: 0
-siblings	: 4
-core id		: 2
-cpu cores	: 2
-apicid		: 5
-initial apicid	: 5
-fpu		: yes
-fpu_exception	: yes
-cpuid level	: 11
-wp		: yes
-flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf pni dtes64 monitor ds_cpl vmx est tm2 ssse3 cx16 xtpr pdcm pcid sse4_1 sse4_2 popcnt lahf_lm ida arat dtherm tpr_shadow vnmi flexpriority ept vpid
-bogomips	: 5053.83
-clflush size	: 64
-cache_alignment	: 64
-address sizes	: 36 bits physical, 48 bits virtual
-power management:
-
-
-
-Memory: 4k page, physical 3900112k(236544k free), swap 0k(0k free)
-
-vm_info: Java HotSpot(TM) 64-Bit Server VM (24.71-b01) for linux-amd64 JRE (1.7.0_71-b14), built on Sep 26 2014 16:41:40 by "java_re" with gcc 4.3.0 20080428 (Red Hat 4.3.0-8)
-
-time: Sun Feb  8 03:10:22 2015
-elapsed time: 0 seconds
-

From d6c93c54eadfc216e29377920f2c4937ae73cd4e Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Mon, 9 Feb 2015 10:20:38 +0800
Subject: [PATCH 11/13] fix bug

---
 .../apache/spark/sql/hive/TableReader.scala   | 21 ++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index 43991a99d6403..0ab069cb91b5e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -25,7 +25,6 @@ import org.apache.hadoop.hive.ql.exec.Utilities
 import org.apache.hadoop.hive.ql.metadata.{Partition => HivePartition, Table => HiveTable}
 import org.apache.hadoop.hive.ql.plan.{PlanUtils, TableDesc}
 import org.apache.hadoop.hive.serde2.Deserializer
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.IdentityConverter
 import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspectorConverters, StructObjectInspector}
 import org.apache.hadoop.hive.serde2.objectinspector.primitive._
 import org.apache.hadoop.io.Writable
@@ -279,14 +278,16 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       mutableRow: MutableRow,
       tableDeser: Deserializer): Iterator[Row] = {
 
-    val soi = HiveShim.getConvertedOI(
-      rawDeser.getObjectInspector,
-      tableDeser.getObjectInspector).asInstanceOf[StructObjectInspector]
-
-    val inputFields = soi.getAllStructFieldRefs
+    val soi = if (rawDeser.getObjectInspector.equals(tableDeser.getObjectInspector)) {
+      rawDeser.getObjectInspector.asInstanceOf[StructObjectInspector]
+    } else {
+      HiveShim.getConvertedOI(
+        rawDeser.getObjectInspector,
+        tableDeser.getObjectInspector).asInstanceOf[StructObjectInspector]
+    }
 
     val (fieldRefs, fieldOrdinals) = nonPartitionKeyAttrs.map { case (attr, ordinal) =>
-      (inputFields.get(ordinal), ordinal)
+      soi.getStructFieldRef(attr.name) -> ordinal
     }.unzip
 
     /**
@@ -329,11 +330,7 @@ private[hive] object HadoopTableReader extends HiveInspectors {
       }
     }
 
-    val converter = if (rawDeser == tableDeser) {
-      new IdentityConverter
-    } else {
-      ObjectInspectorConverters.getConverter(rawDeser.getObjectInspector, soi)
-    }
+    val converter = ObjectInspectorConverters.getConverter(rawDeser.getObjectInspector, soi)
 
     // Map each tuple to a row object
     iterator.map { value =>

From b41d6b9354a6fc6d449315c20f848a47493060fd Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Fri, 13 Mar 2015 19:34:42 +0800
Subject: [PATCH 12/13] fix compile errors

---
 .../spark/sql/hive/InsertIntoHiveTableSuite.scala  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
index 641e496fdc55c..66a3b425edbd4 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/InsertIntoHiveTableSuite.scala
@@ -191,13 +191,13 @@ class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter {
     sql("DROP TABLE hiveTableWithStructValue")
   }
 
-  test("SPARK-5498:partition schema does not match table schema"){
+  test("SPARK-5498:partition schema does not match table schema") {
     val testData = TestHive.sparkContext.parallelize(
-      (1 to 10).map(i => TestData(i, i.toString)))
+      (1 to 10).map(i => TestData(i, i.toString))).toDF()
     testData.registerTempTable("testData")
 
     val testDatawithNull = TestHive.sparkContext.parallelize(
-      (1 to 10).map(i => ThreeCloumntable(i, i.toString,null)))
+      (1 to 10).map(i => ThreeCloumntable(i, i.toString,null))).toDF()
 
     val tmpDir = Files.createTempDir()
     sql(s"CREATE TABLE table_with_partition(key int,value string) PARTITIONED by (ds string) location '${tmpDir.toURI.toString}' ")
@@ -206,25 +206,25 @@ class InsertIntoHiveTableSuite extends QueryTest with BeforeAndAfter {
     // test schema the same between partition and table
     sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
     checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
-      testData.toDataFrame.collect.toSeq
+      testData.collect.toSeq
     )
     
     // test difference type of field
     sql("ALTER TABLE table_with_partition CHANGE COLUMN key key BIGINT")
     checkAnswer(sql("select key,value from table_with_partition where ds='1' "),
-      testData.toDataFrame.collect.toSeq
+      testData.collect.toSeq
     )
 
     // add column to table
     sql("ALTER TABLE table_with_partition ADD COLUMNS(key1 string)")
     checkAnswer(sql("select key,value,key1 from table_with_partition where ds='1' "),
-      testDatawithNull.toDataFrame.collect.toSeq
+      testDatawithNull.collect.toSeq
     )
 
     // change column name to table
     sql("ALTER TABLE table_with_partition CHANGE COLUMN key keynew BIGINT")
     checkAnswer(sql("select keynew,value from table_with_partition where ds='1' "),
-      testData.toDataFrame.collect.toSeq
+      testData.collect.toSeq
     )
 
     sql("DROP TABLE table_with_partition")

From 9c8da744b3347731bca0242f69895bc10bdc1967 Mon Sep 17 00:00:00 2001
From: jeanlyn <jeanlyn92@gmail.com>
Date: Wed, 18 Mar 2015 11:44:58 +0800
Subject: [PATCH 13/13] fix style

---
 .../src/main/scala/org/apache/spark/sql/hive/Shim12.scala  | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index b78c434e0b193..0ed93c2c5b1fa 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -189,7 +189,7 @@ private[hive] object HiveShim {
 
   def getStatsSetupConstRawDataSize = StatsSetupConst.RAW_DATA_SIZE
 
-  def createDefaultDBIfNeeded(context: HiveContext) = {}
+  def createDefaultDBIfNeeded(context: HiveContext) = {  }
 
   def getCommandProcessor(cmd: Array[String], conf: HiveConf) = {
     CommandProcessorFactory.get(cmd(0), conf)
@@ -244,8 +244,9 @@ private[hive] object HiveShim {
     }
   }
 
-  def getConvertedOI(inputOI: ObjectInspector,
-                     outputOI: ObjectInspector): ObjectInspector = {
+  def getConvertedOI(
+      inputOI: ObjectInspector,
+      outputOI: ObjectInspector): ObjectInspector = {
     ObjectInspectorConverters.getConvertedOI(inputOI, outputOI, true)
   }