check with catalyst types

Author: Davies Liu

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala

@@ -38,10 +38,23 @@ trait ExpressionEvalHelper {
 
   protected def checkEvaluation(
       expression: Expression, expected: Any, inputRow: InternalRow = EmptyRow): Unit = {
-    checkEvaluationWithoutCodegen(expression, expected, inputRow)
-    checkEvaluationWithGeneratedMutableProjection(expression, expected, inputRow)
-    checkEvaluationWithGeneratedProjection(expression, expected, inputRow)
-    checkEvaluationWithOptimization(expression, expected, inputRow)
+    val catalystValue = CatalystTypeConverters.convertToCatalyst(expected)
+    checkEvaluationWithoutCodegen(expression, catalystValue, inputRow)
+    checkEvaluationWithGeneratedMutableProjection(expression, catalystValue, inputRow)
+    checkEvaluationWithGeneratedProjection(expression, catalystValue, inputRow)
+    checkEvaluationWithOptimization(expression, catalystValue, inputRow)
+  }
+
+  /**
+   * Check the equality between result of expression and expected value, it will handle
+   * Array[Byte].
+   */
+  protected def checkResult(result: Any, expected: Any): Boolean = {
+    (result, expected) match {
+      case (result: Array[Byte], expected: Array[Byte]) =>
+        java.util.Arrays.equals(result, expected)
+      case _ => result == expected
+    }
   }
 
   protected def evaluate(expression: Expression, inputRow: InternalRow = EmptyRow): Any = {
@@ -55,7 +68,7 @@ trait ExpressionEvalHelper {
     val actual = try evaluate(expression, inputRow) catch {
       case e: Exception => fail(s"Exception evaluating $expression", e)
     }
-    if (actual != expected) {
+    if (!checkResult(actual, expected)) {
       val input = if (inputRow == EmptyRow) "" else s", input: $inputRow"
       fail(s"Incorrect evaluation (codegen off): $expression, " +
         s"actual: $actual, " +
@@ -83,7 +96,7 @@ trait ExpressionEvalHelper {
     }
 
     val actual = plan(inputRow).apply(0)
-    if (actual != expected) {
+    if (!checkResult(actual, expected)) {
       val input = if (inputRow == EmptyRow) "" else s", input: $inputRow"
       fail(s"Incorrect Evaluation: $expression, actual: $actual, expected: $expected$input")
     }
@@ -109,7 +122,7 @@ trait ExpressionEvalHelper {
     }
 
     val actual = plan(inputRow)
-    val expectedRow = new GenericRow(Array[Any](CatalystTypeConverters.convertToCatalyst(expected)))
+    val expectedRow = new GenericRow(Array[Any](expected))
     if (actual.hashCode() != expectedRow.hashCode()) {
       fail(
         s"""

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringFunctionsSuite.scala

@@ -222,9 +222,6 @@ class StringFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
     checkEvaluation(StringLength(regEx), 5, create_row("abdef"))
     checkEvaluation(StringLength(regEx), 0, create_row(""))
     checkEvaluation(StringLength(regEx), null, create_row(null))
-    // TODO currently bug in codegen, let's temporally disable this
-    // checkEvaluation(StringLength(Literal.create(null, StringType)), null, create_row("abdef"))
+    checkEvaluation(StringLength(Literal.create(null, StringType)), null, create_row("abdef"))
   }
-
-
 }

unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java

@@ -17,10 +17,10 @@
 
 package org.apache.spark.unsafe.types;
 
+import javax.annotation.Nullable;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
 import java.util.Arrays;
-import javax.annotation.Nullable;
 
 import org.apache.spark.unsafe.PlatformDependent;
 
@@ -196,10 +196,6 @@ public int compare(final UTF8String other) {
   public boolean equals(final Object other) {
     if (other instanceof UTF8String) {
       return Arrays.equals(bytes, ((UTF8String) other).getBytes());
-    } else if (other instanceof String) {
-      // Used only in unit tests.
-      String s = (String) other;
-      return bytes.length >= s.length() && length() == s.length() && toString().equals(s);
     } else {
       return false;
     }