From 52e29ae92598179e20f7a4bc452976e6527b8140 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 8 Oct 2019 09:34:04 +0300 Subject: [PATCH 1/2] Set spark.sql.legacy.sizeOfNull to false by default --- .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 3d28b5e93a17..fc2cc8041068 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -1865,7 +1865,7 @@ object SQLConf { .doc("If it is set to true, size of null returns -1. This behavior was inherited from Hive. " + "The size function returns null for null input if the flag is disabled.") .booleanConf - .createWithDefault(true) + .createWithDefault(false) val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED = buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled") From a8a0a981b63280e2441146d1502764fb277596b5 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Tue, 8 Oct 2019 09:48:46 +0300 Subject: [PATCH 2/2] Change the Size description --- .../spark/sql/catalyst/expressions/collectionOperations.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 1320648f4cfc..57b692ac1606 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -79,7 +79,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression _FUNC_(expr) - Returns the size of an array or a map. The function returns -1 if its input is null and spark.sql.legacy.sizeOfNull is set to true. If spark.sql.legacy.sizeOfNull is set to false, the function returns null for null input. - By default, the spark.sql.legacy.sizeOfNull parameter is set to true. + By default, the spark.sql.legacy.sizeOfNull parameter is set to false. """, examples = """ Examples: @@ -88,7 +88,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression > SELECT _FUNC_(map('a', 1, 'b', 2)); 2 > SELECT _FUNC_(NULL); - -1 + NULL """) case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes {