From 39abf22f4257acd564f0f29a551751c8a35018d7 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 12 Apr 2017 17:28:31 -0700 Subject: [PATCH 1/3] fix. --- .../apache/spark/sql/catalyst/expressions/ScalaUDF.scala | 9 +++++++++ .../spark/sql/catalyst/expressions/ScalaUDFSuite.scala | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala index 228f4b756c8b..5f533c1defcf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.SparkException +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.types.DataType @@ -45,6 +46,14 @@ case class ScalaUDF( udfName: Option[String] = None) extends Expression with ImplicitCastInputTypes with NonSQLExpression { + // the user-defined functions must be deterministic. + if (!super.deterministic) { + val name = udfName.getOrElse("") + throw new AnalysisException(s"User-defined functions must be deterministic. Name: $name.") + } + + final override def deterministic: Boolean = true + override def nullable: Boolean = true override def toString: String = diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala index 13bd363c8b69..5cc22d780dee 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions import java.util.Locale import org.apache.spark.{SparkException, SparkFunSuite} +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.types.{IntegerType, StringType} class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper { @@ -32,6 +33,13 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(stringUdf, "ax") } + test("non-deterministic children") { + val e = intercept[AnalysisException] { + ScalaUDF((i: Int) => i + 1, IntegerType, Rand(1) :: Nil, Nil, Some("udf")) + }.getMessage + assert(e.contains("User-defined functions must be deterministic. Name: udf.")) + } + test("better error message for NPE") { val udf = ScalaUDF( (s: String) => s.toLowerCase(Locale.ROOT), From 4e59b6715475313653efe1d21f1a2ec270ccfd5a Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 12 Apr 2017 17:40:55 -0700 Subject: [PATCH 2/3] correct it --- .../apache/spark/sql/catalyst/expressions/ScalaUDF.scala | 5 ----- .../spark/sql/catalyst/expressions/ScalaUDFSuite.scala | 8 -------- 2 files changed, 13 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala index 5f533c1defcf..d3ae339707aa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala @@ -47,11 +47,6 @@ case class ScalaUDF( extends Expression with ImplicitCastInputTypes with NonSQLExpression { // the user-defined functions must be deterministic. - if (!super.deterministic) { - val name = udfName.getOrElse("") - throw new AnalysisException(s"User-defined functions must be deterministic. Name: $name.") - } - final override def deterministic: Boolean = true override def nullable: Boolean = true diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala index 5cc22d780dee..13bd363c8b69 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDFSuite.scala @@ -20,7 +20,6 @@ package org.apache.spark.sql.catalyst.expressions import java.util.Locale import org.apache.spark.{SparkException, SparkFunSuite} -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.types.{IntegerType, StringType} class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper { @@ -33,13 +32,6 @@ class ScalaUDFSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(stringUdf, "ax") } - test("non-deterministic children") { - val e = intercept[AnalysisException] { - ScalaUDF((i: Int) => i + 1, IntegerType, Rand(1) :: Nil, Nil, Some("udf")) - }.getMessage - assert(e.contains("User-defined functions must be deterministic. Name: udf.")) - } - test("better error message for NPE") { val udf = ScalaUDF( (s: String) => s.toLowerCase(Locale.ROOT), From 03303a9476354d4dd38c1f4815858441a409aed4 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Wed, 12 Apr 2017 17:42:50 -0700 Subject: [PATCH 3/3] revert --- .../org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala index d3ae339707aa..88525cd6e4cd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.SparkException -import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.types.DataType