Handle null field properly

Author: HeartSaVioR

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/InternalRow.scala

@@ -78,31 +78,33 @@ abstract class InternalRow extends SpecializedGetters with Serializable {
       var newRet: InternalRow = ret
       if (field.ne(newField)) {
         if (newRet == null) newRet = this.copy()
-        if (newField != null) {
-          newRet.update(idx, newField)
-        } else {
-          newRet.setNullAt(idx)
-        }
+        newRet.update(idx, newField)
       }
       newRet
     }
 
     var ret: InternalRow = null
     dataType.map(_.dataType).zipWithIndex.foreach {
       case (ty: StructType, idx) =>
-        val field = getStruct(idx, ty.size)
-        val newField = field.copyUnsafeData(ty)
-        ret = updateRetIfNecessary(ret, field, newField, idx)
+        if (!isNullAt(idx)) {
+          val field = getStruct(idx, ty.size)
+          val newField = field.copyUnsafeData(ty)
+          ret = updateRetIfNecessary(ret, field, newField, idx)
+        }
 
       case (ty: ArrayType, idx) =>
-        val field = getArray(idx)
-        val newField = field.copyUnsafeData(ty)
-        ret = updateRetIfNecessary(ret, field, newField, idx)
+        if (!isNullAt(idx)) {
+          val field = getArray(idx)
+          val newField = field.copyUnsafeData(ty)
+          ret = updateRetIfNecessary(ret, field, newField, idx)
+        }
 
       case (ty: MapType, idx) =>
-        val field = getMap(idx)
-        val newField = field.copyUnsafeData(ty)
-        ret = updateRetIfNecessary(ret, field, newField, idx)
+        if (!isNullAt(idx)) {
+          val field = getMap(idx)
+          val newField = field.copyUnsafeData(ty)
+          ret = updateRetIfNecessary(ret, field, newField, idx)
+        }
 
       case _ =>
     }

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ArrayData.scala

@@ -83,11 +83,7 @@ abstract class ArrayData extends SpecializedGetters with Serializable {
       var newRet: ArrayData = ret
       if (field.ne(newField)) {
         if (newRet == null) newRet = this.copy()
-        if (newField != null) {
-          newRet.update(idx, newField)
-        } else {
-          newRet.setNullAt(idx)
-        }
+        newRet.update(idx, newField)
       }
       newRet
     }
@@ -98,9 +94,11 @@ abstract class ArrayData extends SpecializedGetters with Serializable {
         val len = numElements()
         var i = 0
         while (i < len) {
-          val field = getStruct(i, ty.size)
-          val newField = field.copyUnsafeData(ty)
-          ret = updateRetIfNecessary(ret, field, newField, i)
+          if (!isNullAt(i)) {
+            val field = getStruct(i, ty.size)
+            val newField = field.copyUnsafeData(ty)
+            ret = updateRetIfNecessary(ret, field, newField, i)
+          }
           i += 1
         }
         if (ret != null) ret else this
@@ -109,9 +107,11 @@ abstract class ArrayData extends SpecializedGetters with Serializable {
         val len = numElements()
         var i = 0
         while (i < len) {
-          val field = getArray(i)
-          val newField = field.copyUnsafeData(ty)
-          ret = updateRetIfNecessary(ret, field, newField, i)
+          if (!isNullAt(i)) {
+            val field = getArray(i)
+            val newField = field.copyUnsafeData(ty)
+            ret = updateRetIfNecessary(ret, field, newField, i)
+          }
           i += 1
         }
         if (ret != null) ret else this
@@ -120,9 +120,11 @@ abstract class ArrayData extends SpecializedGetters with Serializable {
         val len = numElements()
         var i = 0
         while (i < len) {
-          val field = getMap(i)
-          val newField = field.copyUnsafeData(ty)
-          ret = updateRetIfNecessary(ret, field, newField, i)
+          if (!isNullAt(i)) {
+            val field = getMap(i)
+            val newField = field.copyUnsafeData(ty)
+            ret = updateRetIfNecessary(ret, field, newField, i)
+          }
           i += 1
         }
         if (ret != null) ret else this