From e25bfdaf52b53f76bfc7f2f5e5a96bd9a127e615 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Wed, 25 Sep 2024 19:36:50 +0900 Subject: [PATCH 01/14] [SPARK-49768][SQL] Provide error Conditions for make_date/make_timestamp errors _LEGACY_ERROR_TEMP_2000 --- .../src/main/resources/error/error-conditions.json | 11 ++++++----- .../spark/sql/errors/QueryExecutionErrors.scala | 4 ++-- .../catalyst/expressions/DateExpressionsSuite.scala | 2 +- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 2 +- .../resources/sql-tests/results/ansi/date.sql.out | 4 ++-- .../sql-tests/results/ansi/timestamp.sql.out | 6 +++--- .../sql-tests/results/postgreSQL/date.sql.out | 6 +++--- .../results/timestampNTZ/timestamp-ansi.sql.out | 6 +++--- 8 files changed, 21 insertions(+), 20 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index e83202d9e5ee..1bdd6f27b38c 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -70,6 +70,12 @@ ], "sqlState" : "42000" }, + "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS" : { + "message" : [ + "ANSI mode error: . If necessary set to false to bypass this error." + ], + "sqlState" : "22008" + }, "ARITHMETIC_OVERFLOW" : { "message" : [ ". If necessary set to \"false\" to bypass this error." @@ -6691,11 +6697,6 @@ "Sinks cannot request distribution and ordering in continuous execution mode." ] }, - "_LEGACY_ERROR_TEMP_2000" : { - "message" : [ - ". If necessary set to false to bypass this error." - ] - }, "_LEGACY_ERROR_TEMP_2003" : { "message" : [ "Unsuccessful try to zip maps with unique keys due to exceeding the array size limit ." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 4a23e9766fc5..ab225e19ab1f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -279,7 +279,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def ansiDateTimeError(e: Exception): SparkDateTimeException = { new SparkDateTimeException( - errorClass = "_LEGACY_ERROR_TEMP_2000", + errorClass = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( "message" -> e.getMessage, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), @@ -289,7 +289,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def ansiIllegalArgumentError(message: String): SparkIllegalArgumentException = { new SparkIllegalArgumentException( - errorClass = "_LEGACY_ERROR_TEMP_2000", + errorClass = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( "message" -> message, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 21ae35146282..d38ce68a617e 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -436,7 +436,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) { checkErrorInExpression[SparkIllegalArgumentException]( DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))), - "_LEGACY_ERROR_TEMP_2000", + "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", Map("message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 96aaf13052b0..9a621bc7ebbd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -542,7 +542,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { checkError( exception = intercept[SparkIllegalArgumentException]( dateAddInterval(input, new CalendarInterval(36, 47, 1))), - condition = "_LEGACY_ERROR_TEMP_2000", + condition = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", parameters = Map( "message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 67cd23faf255..81878180bcc6 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -53,7 +53,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" @@ -68,7 +68,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): 33" diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index d7a58e321b0f..b6a0203d9a03 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -154,7 +154,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" @@ -185,7 +185,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" @@ -200,7 +200,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 8caf8c54b9f3..7b8a4493c930 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -687,7 +687,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid date 'FEBRUARY 30'" @@ -702,7 +702,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" @@ -717,7 +717,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): -1" diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index cd94674d2bf2..1449bf1c5c44 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -154,7 +154,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" @@ -185,7 +185,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" @@ -200,7 +200,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "_LEGACY_ERROR_TEMP_2000", + "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" From a7d8b7e3580cb624ab5719c8ff46442519f61388 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 26 Sep 2024 09:18:32 +0900 Subject: [PATCH 02/14] Rename --- .../src/main/resources/error/error-conditions.json | 12 ++++++------ .../spark/sql/errors/QueryExecutionErrors.scala | 4 ++-- .../catalyst/expressions/DateExpressionsSuite.scala | 2 +- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 2 +- .../resources/sql-tests/results/ansi/date.sql.out | 4 ++-- .../sql-tests/results/ansi/timestamp.sql.out | 6 +++--- .../sql-tests/results/postgreSQL/date.sql.out | 6 +++--- .../results/timestampNTZ/timestamp-ansi.sql.out | 6 +++--- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 1bdd6f27b38c..6f3b012f2093 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -70,12 +70,6 @@ ], "sqlState" : "42000" }, - "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS" : { - "message" : [ - "ANSI mode error: . If necessary set to false to bypass this error." - ], - "sqlState" : "22008" - }, "ARITHMETIC_OVERFLOW" : { "message" : [ ". If necessary set to \"false\" to bypass this error." @@ -1089,6 +1083,12 @@ ], "sqlState" : "22008" }, + "DATE_TIME_FIELD_OUT_OF_BOUNDS" : { + "message" : [ + "ANSI mode error: . If necessary set to false to bypass this error." + ], + "sqlState" : "22008" + }, "DECIMAL_PRECISION_EXCEEDS_MAX_PRECISION" : { "message" : [ "Decimal precision exceeds max precision ." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index ab225e19ab1f..065a7cb0ac33 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -279,7 +279,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def ansiDateTimeError(e: Exception): SparkDateTimeException = { new SparkDateTimeException( - errorClass = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( "message" -> e.getMessage, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), @@ -289,7 +289,7 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def ansiIllegalArgumentError(message: String): SparkIllegalArgumentException = { new SparkIllegalArgumentException( - errorClass = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( "message" -> message, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index d38ce68a617e..201f57a70bb3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -436,7 +436,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) { checkErrorInExpression[SparkIllegalArgumentException]( DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))), - "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATE_TIME_FIELD_OUT_OF_BOUNDS", Map("message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 9a621bc7ebbd..be494e3e4824 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -542,7 +542,7 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { checkError( exception = intercept[SparkIllegalArgumentException]( dateAddInterval(input, new CalendarInterval(36, 47, 1))), - condition = "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + condition = "DATE_TIME_FIELD_OUT_OF_BOUNDS", parameters = Map( "message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 81878180bcc6..1d93363fa83c 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -53,7 +53,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" @@ -68,7 +68,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): 33" diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index b6a0203d9a03..c27baad353ef 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -154,7 +154,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" @@ -185,7 +185,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" @@ -200,7 +200,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 7b8a4493c930..5b15816da69f 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -687,7 +687,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid date 'FEBRUARY 30'" @@ -702,7 +702,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" @@ -717,7 +717,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): -1" diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index 1449bf1c5c44..97282257dcce 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -154,7 +154,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" @@ -185,7 +185,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" @@ -200,7 +200,7 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "ANSI_DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" From fe025ab0b098dabfe0c71d67573cf5d2a743a490 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 26 Sep 2024 09:40:17 +0900 Subject: [PATCH 03/14] Use existing error message --- common/utils/src/main/resources/error/error-conditions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 6f3b012f2093..08edbdfded3c 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1085,7 +1085,7 @@ }, "DATE_TIME_FIELD_OUT_OF_BOUNDS" : { "message" : [ - "ANSI mode error: . If necessary set to false to bypass this error." + ". If necessary set to false to bypass this error." ], "sqlState" : "22008" }, From c8cb363cc40495c9b75bd70b57b29bdbb7afe133 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 30 Sep 2024 11:47:53 +0900 Subject: [PATCH 04/14] Adjusted comments --- .../resources/error/error-conditions.json | 8 ++++++- .../org/apache/spark/SparkException.scala | 23 ++++++++++++++++--- .../sql/catalyst/util/DateTimeUtils.scala | 3 +-- .../sql/errors/QueryExecutionErrors.scala | 9 ++++---- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 08edbdfded3c..deb58b26f4e2 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1085,7 +1085,7 @@ }, "DATE_TIME_FIELD_OUT_OF_BOUNDS" : { "message" : [ - ". If necessary set to false to bypass this error." + "Invalid value for datetime field. If necessary set to false to bypass this error." ], "sqlState" : "22008" }, @@ -4788,6 +4788,12 @@ ], "sqlState" : "0A000" }, + "UNSUPPORTED_DATETIME_UNIT_ADDITION" : { + "message" : [ + "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. If necessary set to false to bypass this error." + ], + "sqlState" : "22008" + }, "UNSUPPORTED_DEFAULT_VALUE" : { "message" : [ "DEFAULT column values is not supported." diff --git a/common/utils/src/main/scala/org/apache/spark/SparkException.scala b/common/utils/src/main/scala/org/apache/spark/SparkException.scala index 398cb1fad672..974c5875de23 100644 --- a/common/utils/src/main/scala/org/apache/spark/SparkException.scala +++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala @@ -306,8 +306,9 @@ private[spark] class SparkDateTimeException private( message: String, errorClass: Option[String], messageParameters: Map[String, String], - context: Array[QueryContext]) - extends DateTimeException(message) with SparkThrowable { + context: Array[QueryContext], + cause: Option[Throwable]) + extends DateTimeException(message, cause.orNull) with SparkThrowable { def this( errorClass: String, @@ -318,7 +319,23 @@ private[spark] class SparkDateTimeException private( SparkThrowableHelper.getMessage(errorClass, messageParameters, summary), Option(errorClass), messageParameters, - context + context, + cause = null + ) + } + + def this( + errorClass: String, + messageParameters: Map[String, String], + context: Array[QueryContext], + summary: String, + cause: Option[Throwable]) = { + this( + SparkThrowableHelper.getMessage(errorClass, messageParameters, summary), + Option(errorClass), + messageParameters, + context, + cause ) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index e27ce29fc231..8c20d0bc252a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -304,8 +304,7 @@ object DateTimeUtils extends SparkDateTimeUtils { start: Int, interval: CalendarInterval): Int = { if (interval.microseconds != 0) { - throw QueryExecutionErrors.ansiIllegalArgumentError( - "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date") + throw QueryExecutionErrors.ansiIllegalArgumentError() } val ld = daysToLocalDate(start).plusMonths(interval.months).plusDays(interval.days) localDateToDays(ld) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 065a7cb0ac33..d26da863aae1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -281,17 +281,16 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE new SparkDateTimeException( errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( - "message" -> e.getMessage, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), context = Array.empty, - summary = "") + summary = "", + cause = Some(e)) } - def ansiIllegalArgumentError(message: String): SparkIllegalArgumentException = { + def ansiIllegalArgumentError(): SparkIllegalArgumentException = { new SparkIllegalArgumentException( - errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", + errorClass = "UNSUPPORTED_DATETIME_UNIT_ADDITION", messageParameters = Map( - "message" -> message, "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) } From b71e1853451d6e4dc105a3e1f6f3973c3afbfb82 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 30 Sep 2024 13:52:44 +0900 Subject: [PATCH 05/14] Regen golden files --- .../main/scala/org/apache/spark/SparkException.scala | 4 ++-- .../resources/sql-tests/results/ansi/date.sql.out | 8 ++++---- .../sql-tests/results/ansi/timestamp.sql.out | 12 ++++++------ .../sql-tests/results/postgreSQL/date.sql.out | 12 ++++++------ .../results/timestampNTZ/timestamp-ansi.sql.out | 12 ++++++------ 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/common/utils/src/main/scala/org/apache/spark/SparkException.scala b/common/utils/src/main/scala/org/apache/spark/SparkException.scala index 974c5875de23..fcaee787fd8d 100644 --- a/common/utils/src/main/scala/org/apache/spark/SparkException.scala +++ b/common/utils/src/main/scala/org/apache/spark/SparkException.scala @@ -320,7 +320,7 @@ private[spark] class SparkDateTimeException private( Option(errorClass), messageParameters, context, - cause = null + cause = None ) } @@ -335,7 +335,7 @@ private[spark] class SparkDateTimeException private( Option(errorClass), messageParameters, context, - cause + cause.orElse(None) ) } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 1d93363fa83c..914351bd04ad 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -54,9 +54,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -69,9 +69,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): 33" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index c27baad353ef..20f383629ea8 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -155,9 +155,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -186,9 +186,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -201,9 +201,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 5b15816da69f..5266337e36b2 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -688,9 +688,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid date 'FEBRUARY 30'" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -703,9 +703,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for MonthOfYear (valid values 1 - 12): 13" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -718,9 +718,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for DayOfMonth (valid values 1 - 28/31): -1" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index 97282257dcce..c26abc654b8a 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -155,9 +155,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 61" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -186,9 +186,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 99" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } @@ -201,9 +201,9 @@ struct<> org.apache.spark.SparkDateTimeException { "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "message" : "Invalid value for SecondOfMinute (valid values 0 - 59): 999" + "ansiConfig" : "\"spark.sql.ansi.enabled\"" } } From 27fea85738ce18fdad22cc8e2cbe58218fd85a3f Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 30 Sep 2024 19:28:54 +0900 Subject: [PATCH 06/14] fix test --- .../expressions/DateExpressionsSuite.scala | 45 +++++++++++-------- .../catalyst/util/DateTimeUtilsSuite.scala | 3 +- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 201f57a70bb3..54994c6fb670 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -436,10 +436,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) { checkErrorInExpression[SparkIllegalArgumentException]( DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", - Map("message" -> - "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", - "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + "UNSUPPORTED_DATETIME_UNIT_ADDITION", + Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { @@ -1140,12 +1138,18 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { // ansi test withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - checkExceptionInExpression[DateTimeException](MakeDate(Literal(Int.MaxValue), Literal(13), - Literal(19)), EmptyRow, "Invalid value for Year") - checkExceptionInExpression[DateTimeException](MakeDate(Literal(2019), - Literal(13), Literal(19)), EmptyRow, "Invalid value for Month") - checkExceptionInExpression[DateTimeException](MakeDate(Literal(2019), Literal(7), - Literal(32)), EmptyRow, "Invalid value for Day") + checkErrorInExpression[SparkDateTimeException]( + MakeDate(Literal(Int.MaxValue), Literal(13), Literal(19)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + checkErrorInExpression[SparkDateTimeException]( + MakeDate(Literal(2019), Literal(13), Literal(19)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + checkErrorInExpression[SparkDateTimeException]( + MakeDate(Literal(2019), Literal(7), Literal(32)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } // non-ansi test @@ -1183,18 +1187,21 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(makeTimestampExpr.copy(timezone = None), expected) Seq( - (makeTimestampExpr.copy(year = Literal(Int.MaxValue)), "Invalid value for Year"), - (makeTimestampExpr.copy(month = Literal(13)), "Invalid value for Month"), - (makeTimestampExpr.copy(day = Literal(32)), "Invalid value for Day"), - (makeTimestampExpr.copy(hour = Literal(25)), "Invalid value for Hour"), - (makeTimestampExpr.copy(min = Literal(65)), "Invalid value for Min"), - (makeTimestampExpr.copy(sec = Literal(Decimal( - BigDecimal(70.0), 16, 6))), "Invalid value for Second") + makeTimestampExpr.copy(year = Literal(Int.MaxValue)), + makeTimestampExpr.copy(month = Literal(13)), + makeTimestampExpr.copy(day = Literal(32)), + makeTimestampExpr.copy(hour = Literal(25)), + makeTimestampExpr.copy(min = Literal(65)), + makeTimestampExpr.copy(sec = Literal(Decimal( + BigDecimal(70.0), 16, 6))) ).foreach { entry => if (ansi) { - checkExceptionInExpression[DateTimeException](entry._1, EmptyRow, entry._2) + checkErrorInExpression[SparkDateTimeException]( + entry, + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } else { - checkEvaluation(entry._1, null) + checkEvaluation(entry, null) } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index be494e3e4824..28fc2f125cbe 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -542,9 +542,8 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { checkError( exception = intercept[SparkIllegalArgumentException]( dateAddInterval(input, new CalendarInterval(36, 47, 1))), - condition = "DATE_TIME_FIELD_OUT_OF_BOUNDS", + condition = "UNSUPPORTED_DATETIME_UNIT_ADDITION", parameters = Map( - "message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } From 5bf98feb392db6feaa6aa298173a8e07b528ac13 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Wed, 2 Oct 2024 09:27:18 +0900 Subject: [PATCH 07/14] switched to INTERNAL_ERROR --- .../src/main/resources/error/error-conditions.json | 6 ------ .../spark/sql/errors/QueryExecutionErrors.scala | 10 +++++----- .../catalyst/expressions/DateExpressionsSuite.scala | 10 +++++++--- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 11 +++++++---- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index deb58b26f4e2..b44d33bceb10 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -4788,12 +4788,6 @@ ], "sqlState" : "0A000" }, - "UNSUPPORTED_DATETIME_UNIT_ADDITION" : { - "message" : [ - "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. If necessary set to false to bypass this error." - ], - "sqlState" : "22008" - }, "UNSUPPORTED_DEFAULT_VALUE" : { "message" : [ "DEFAULT column values is not supported." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index d26da863aae1..56bd788e53f6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -287,11 +287,11 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE cause = Some(e)) } - def ansiIllegalArgumentError(): SparkIllegalArgumentException = { - new SparkIllegalArgumentException( - errorClass = "UNSUPPORTED_DATETIME_UNIT_ADDITION", - messageParameters = Map( - "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) + def ansiIllegalArgumentError(): SparkException = { + SparkException.internalError( + "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + + s"If necessary set ${toSQLConf(SQLConf.ANSI_ENABLED.key)} to false to bypass this error" + ) } def overflowInSumOfDecimalError(context: QueryContext): ArithmeticException = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 54994c6fb670..8e3df22aeb59 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -434,10 +434,14 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) { - checkErrorInExpression[SparkIllegalArgumentException]( + checkErrorInExpression[SparkException]( DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))), - "UNSUPPORTED_DATETIME_UNIT_ADDITION", - Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + condition = "INTERNAL_ERROR", + parameters = Map( + "message" -> + ("Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + + "If necessary set \"spark.sql.ansi.enabled\" to false to bypass this error")) + ) } withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index 28fc2f125cbe..d2bdfe0c2499 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers._ -import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException} +import org.apache.spark.{SparkException, SparkFunSuite, SparkIllegalArgumentException} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ @@ -540,11 +540,14 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15)) assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28)) checkError( - exception = intercept[SparkIllegalArgumentException]( + exception = intercept[SparkException]( dateAddInterval(input, new CalendarInterval(36, 47, 1))), - condition = "UNSUPPORTED_DATETIME_UNIT_ADDITION", + condition = "INTERNAL_ERROR", parameters = Map( - "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + "message" -> + ("Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + + "If necessary set \"spark.sql.ansi.enabled\" to false to bypass this error")) + ) } test("timestamp add interval") { From 260145e4479f9d89a2e5f2a3913bf220df1230df Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Wed, 2 Oct 2024 10:09:13 +0900 Subject: [PATCH 08/14] Added more informations for error message & regen --- .../resources/error/error-conditions.json | 2 +- .../sql/errors/QueryExecutionErrors.scala | 29 +++++++++++++++++-- .../expressions/DateExpressionsSuite.scala | 21 ++++++++++++-- .../sql-tests/results/ansi/date.sql.out | 10 +++++-- .../sql-tests/results/ansi/timestamp.sql.out | 15 ++++++++-- .../sql-tests/results/postgreSQL/date.sql.out | 18 ++++++++---- .../timestampNTZ/timestamp-ansi.sql.out | 15 ++++++++-- 7 files changed, 90 insertions(+), 20 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index b44d33bceb10..2a48122d8c5d 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1085,7 +1085,7 @@ }, "DATE_TIME_FIELD_OUT_OF_BOUNDS" : { "message" : [ - "Invalid value for datetime field. If necessary set to false to bypass this error." + "The value '' you entered for is not valid. Please provide a value between . To disable strict validation, you can turn off ANSI mode by setting to false." ], "sqlState" : "22008" }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 56bd788e53f6..62e30bb7fd40 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -278,13 +278,38 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } def ansiDateTimeError(e: Exception): SparkDateTimeException = { + def extractDateTimeErrorInfo(e: Exception): (String, String, String) = { + val errorMessage = e.getMessage + + val pattern = "Invalid value for ([A-Za-z]+) \\(valid values (.+)\\): (.+)".r + + errorMessage match { + case pattern(field, range, badValue) => + val unit = field match { + case "Year" => "YEAR" + case "MonthOfYear" => "MONTH" + case "DayOfMonth" => "DAY" + case "Hour" => "HOUR" + case "Minute" => "MINUTE" + case "Second" => "SECOND" + case _ => field + } + (unit, range, badValue) + } + } + val (unit, range, badValue) = extractDateTimeErrorInfo(e) new SparkDateTimeException( errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( - "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), + "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key), + "unit" -> unit, + "range" -> range, + "badValue" -> badValue + ), context = Array.empty, summary = "", - cause = Some(e)) + cause = Some(e) + ) } def ansiIllegalArgumentError(): SparkException = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 8e3df22aeb59..27a5af541dbc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1145,15 +1145,30 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(Int.MaxValue), Literal(13), Literal(19)), "DATE_TIME_FIELD_OUT_OF_BOUNDS", - Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "YEAR", + "range" -> "-999999999 - 999999999", + "badValue" -> "2147483647") + ) checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(2019), Literal(13), Literal(19)), "DATE_TIME_FIELD_OUT_OF_BOUNDS", - Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "MONTH", + "range" -> "1 - 12", + "badValue" -> "13") + ) checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(2019), Literal(7), Literal(32)), "DATE_TIME_FIELD_OUT_OF_BOUNDS", - Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "DAY", + "range" -> "1 - 28/31", + "badValue" -> "32") + ) } // non-ansi test diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 914351bd04ad..82d1b0097074 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -56,7 +56,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "13", + "range" : "1 - 12", + "unit" : "MONTH" } } @@ -71,7 +74,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "33", + "range" : "1 - 28/31", + "unit" : "DAY" } } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index 20f383629ea8..d77144153efa 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -157,7 +157,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "61", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } @@ -188,7 +191,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "99", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } @@ -203,7 +209,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "999", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 5266337e36b2..8c97978ab2a0 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -685,12 +685,12 @@ select make_date(2013, 2, 30) -- !query schema struct<> -- !query output -org.apache.spark.SparkDateTimeException +org.apache.spark.SparkException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", - "sqlState" : "22008", + "errorClass" : "INTERNAL_ERROR", + "sqlState" : "XX000", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "message" : "The Spark SQL phase optimization failed with an internal error. You hit a bug in Spark or the Spark plugins you use. Please, report this bug to the corresponding communities or vendors, and provide the full stack trace." } } @@ -705,7 +705,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "13", + "range" : "1 - 12", + "unit" : "MONTH" } } @@ -720,7 +723,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "-1", + "range" : "1 - 28/31", + "unit" : "DAY" } } diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index c26abc654b8a..d19a3e5a1205 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -157,7 +157,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "61", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } @@ -188,7 +191,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "99", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } @@ -203,7 +209,10 @@ org.apache.spark.SparkDateTimeException "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"" + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "999", + "range" : "0 - 59", + "unit" : "SecondOfMinute" } } From 3ea12a4b6fca7f8b599467b55887563740029576 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Wed, 2 Oct 2024 14:46:38 +0900 Subject: [PATCH 09/14] fix test --- .../expressions/DateExpressionsSuite.scala | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 27a5af541dbc..3c784f9c4634 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1214,14 +1214,64 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { makeTimestampExpr.copy(sec = Literal(Decimal( BigDecimal(70.0), 16, 6))) ).foreach { entry => - if (ansi) { - checkErrorInExpression[SparkDateTimeException]( - entry, - "DATE_TIME_FIELD_OUT_OF_BOUNDS", - Map("ansiConfig" -> "\"spark.sql.ansi.enabled\"")) - } else { - checkEvaluation(entry, null) - } + if (!ansi) checkEvaluation(entry, null) + } + + if (ansi) { + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(year = Literal(Int.MaxValue)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "YEAR", + "range" -> "-999999999 - 999999999", + "badValue" -> "2147483647") + ) + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(month = Literal(13)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "MONTH", + "range" -> "1 - 12", + "badValue" -> "13") + ) + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(day = Literal(32)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "DAY", + "range" -> "1 - 28/31", + "badValue" -> "32") + ) + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(hour = Literal(25)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "HourOfDay", + "range" -> "0 - 23", + "badValue" -> "25") + ) + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(min = Literal(65)), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "MinuteOfHour", + "range" -> "0 - 59", + "badValue" -> "65") + ) + checkErrorInExpression[SparkDateTimeException]( + makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(70.0), 16, 6))), + "DATE_TIME_FIELD_OUT_OF_BOUNDS", + Map( + "ansiConfig" -> "\"spark.sql.ansi.enabled\"", + "unit" -> "SecondOfMinute", + "range" -> "0 - 59", + "badValue" -> "70") + ) } makeTimestampExpr = MakeTimestamp(Literal(2019), Literal(6), Literal(30), From 60fb678ffcd0125ab7dad9517581d6bff5dc15df Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 3 Oct 2024 11:38:25 +0900 Subject: [PATCH 10/14] Adjusted comments --- .../sql/errors/QueryExecutionErrors.scala | 10 ++++---- .../expressions/DateExpressionsSuite.scala | 24 +++++++++---------- .../sql-tests/results/ansi/date.sql.out | 4 ++-- .../sql-tests/results/ansi/timestamp.sql.out | 12 +++++----- .../sql-tests/results/postgreSQL/date.sql.out | 4 ++-- .../timestampNTZ/timestamp-ansi.sql.out | 12 +++++----- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 62e30bb7fd40..2018e78ae744 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -289,12 +289,12 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE case "Year" => "YEAR" case "MonthOfYear" => "MONTH" case "DayOfMonth" => "DAY" - case "Hour" => "HOUR" - case "Minute" => "MINUTE" - case "Second" => "SECOND" - case _ => field + case "HourOfDay" => "HOUR" + case "MinuteOfHour" => "MINUTE" + case "SecondOfMinute" => "SECOND" } - (unit, range, badValue) + val formattedRange = range.replace(" - ", " ... ") + (unit, formattedRange, badValue) } } val (unit, range, badValue) = extractDateTimeErrorInfo(e) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 3c784f9c4634..96d1072a2c84 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1148,7 +1148,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "YEAR", - "range" -> "-999999999 - 999999999", + "range" -> "-999999999 ... 999999999", "badValue" -> "2147483647") ) checkErrorInExpression[SparkDateTimeException]( @@ -1157,7 +1157,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "MONTH", - "range" -> "1 - 12", + "range" -> "1 ... 12", "badValue" -> "13") ) checkErrorInExpression[SparkDateTimeException]( @@ -1166,7 +1166,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "DAY", - "range" -> "1 - 28/31", + "range" -> "1 ... 28/31", "badValue" -> "32") ) } @@ -1224,7 +1224,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "YEAR", - "range" -> "-999999999 - 999999999", + "range" -> "-999999999 ... 999999999", "badValue" -> "2147483647") ) checkErrorInExpression[SparkDateTimeException]( @@ -1233,7 +1233,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "MONTH", - "range" -> "1 - 12", + "range" -> "1 ... 12", "badValue" -> "13") ) checkErrorInExpression[SparkDateTimeException]( @@ -1242,7 +1242,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "DAY", - "range" -> "1 - 28/31", + "range" -> "1 ... 28/31", "badValue" -> "32") ) checkErrorInExpression[SparkDateTimeException]( @@ -1250,8 +1250,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { "DATE_TIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", - "unit" -> "HourOfDay", - "range" -> "0 - 23", + "unit" -> "HOUR", + "range" -> "0 ... 23", "badValue" -> "25") ) checkErrorInExpression[SparkDateTimeException]( @@ -1259,8 +1259,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { "DATE_TIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", - "unit" -> "MinuteOfHour", - "range" -> "0 - 59", + "unit" -> "MINUTE", + "range" -> "0 ... 59", "badValue" -> "65") ) checkErrorInExpression[SparkDateTimeException]( @@ -1268,8 +1268,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { "DATE_TIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", - "unit" -> "SecondOfMinute", - "range" -> "0 - 59", + "unit" -> "SECOND", + "range" -> "0 ... 59", "badValue" -> "70") ) } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 82d1b0097074..0982aeec873c 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -58,7 +58,7 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "13", - "range" : "1 - 12", + "range" : "1 ... 12", "unit" : "MONTH" } } @@ -76,7 +76,7 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "33", - "range" : "1 - 28/31", + "range" : "1 ... 28/31", "unit" : "DAY" } } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index d77144153efa..0456df781934 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -159,8 +159,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "61", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } @@ -193,8 +193,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "99", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } @@ -211,8 +211,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "999", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 8c97978ab2a0..8e99ae703d83 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -707,7 +707,7 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "13", - "range" : "1 - 12", + "range" : "1 ... 12", "unit" : "MONTH" } } @@ -725,7 +725,7 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "-1", - "range" : "1 - 28/31", + "range" : "1 ... 28/31", "unit" : "DAY" } } diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index d19a3e5a1205..2ebbc33d5bb2 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -159,8 +159,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "61", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } @@ -193,8 +193,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "99", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } @@ -211,8 +211,8 @@ org.apache.spark.SparkDateTimeException "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", "badValue" : "999", - "range" : "0 - 59", - "unit" : "SecondOfMinute" + "range" : "0 ... 59", + "unit" : "SECOND" } } From 83dd8e7506f47118ddda765e08e106ae5c1be609 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 3 Oct 2024 12:18:45 +0900 Subject: [PATCH 11/14] Handling the Invalid date error --- .../spark/sql/errors/QueryExecutionErrors.scala | 7 +++++-- .../sql-tests/results/postgreSQL/date.sql.out | 11 +++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 2018e78ae744..fac2fd5dd6f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -281,10 +281,11 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE def extractDateTimeErrorInfo(e: Exception): (String, String, String) = { val errorMessage = e.getMessage - val pattern = "Invalid value for ([A-Za-z]+) \\(valid values (.+)\\): (.+)".r + val valuePattern = "Invalid value for ([A-Za-z]+) \\(valid values (.+)\\): (.+)".r + val datePattern = "Invalid date '[A-Z]+ ([0-9]+)'".r errorMessage match { - case pattern(field, range, badValue) => + case valuePattern(field, range, badValue) => val unit = field match { case "Year" => "YEAR" case "MonthOfYear" => "MONTH" @@ -295,6 +296,8 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } val formattedRange = range.replace(" - ", " ... ") (unit, formattedRange, badValue) + case datePattern(badDate) => + ("DAY", "1 ... 28/31", badDate) } } val (unit, range, badValue) = extractDateTimeErrorInfo(e) diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 8e99ae703d83..1af862d6aab2 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -685,12 +685,15 @@ select make_date(2013, 2, 30) -- !query schema struct<> -- !query output -org.apache.spark.SparkException +org.apache.spark.SparkDateTimeException { - "errorClass" : "INTERNAL_ERROR", - "sqlState" : "XX000", + "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "sqlState" : "22008", "messageParameters" : { - "message" : "The Spark SQL phase optimization failed with an internal error. You hit a bug in Spark or the Spark plugins you use. Please, report this bug to the corresponding communities or vendors, and provide the full stack trace." + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "badValue" : "30", + "range" : "1 ... 28/31", + "unit" : "DAY" } } From 5ec18e87bb8d2e501bc03a89fd1993fd90edb238 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 3 Oct 2024 22:54:15 +0900 Subject: [PATCH 12/14] Fix error message format --- .../resources/error/error-conditions.json | 8 ++--- .../sql/errors/QueryExecutionErrors.scala | 4 +-- .../expressions/DateExpressionsSuite.scala | 36 +++++++++---------- .../sql-tests/results/ansi/date.sql.out | 8 ++--- .../sql-tests/results/ansi/timestamp.sql.out | 12 +++---- .../sql-tests/results/postgreSQL/date.sql.out | 12 +++---- .../timestampNTZ/timestamp-ansi.sql.out | 12 +++---- 7 files changed, 46 insertions(+), 46 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 4f4e8e0c6bb3..68aac83f2d29 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1087,15 +1087,15 @@ ], "sqlState" : "42K03" }, - "DATETIME_OVERFLOW" : { + "DATETIME_FIELD_OUT_OF_BOUNDS" : { "message" : [ - "Datetime operation overflow: ." + "The value you entered for is not valid. Please provide a value between . To disable strict validation, you can turn off ANSI mode by setting to \"false\"." ], "sqlState" : "22008" }, - "DATE_TIME_FIELD_OUT_OF_BOUNDS" : { + "DATETIME_OVERFLOW" : { "message" : [ - "The value '' you entered for is not valid. Please provide a value between . To disable strict validation, you can turn off ANSI mode by setting to false." + "Datetime operation overflow: ." ], "sqlState" : "22008" }, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index fac2fd5dd6f9..4e62a8427573 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -302,12 +302,12 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE } val (unit, range, badValue) = extractDateTimeErrorInfo(e) new SparkDateTimeException( - errorClass = "DATE_TIME_FIELD_OUT_OF_BOUNDS", + errorClass = "DATETIME_FIELD_OUT_OF_BOUNDS", messageParameters = Map( "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key), "unit" -> unit, "range" -> range, - "badValue" -> badValue + "badValue" -> toSQLValue(badValue) ), context = Array.empty, summary = "", diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 96d1072a2c84..4a7135151a74 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1144,30 +1144,30 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(Int.MaxValue), Literal(13), Literal(19)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "YEAR", "range" -> "-999999999 ... 999999999", - "badValue" -> "2147483647") + "badValue" -> "'2147483647'") ) checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(2019), Literal(13), Literal(19)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "MONTH", "range" -> "1 ... 12", - "badValue" -> "13") + "badValue" -> "'13'") ) checkErrorInExpression[SparkDateTimeException]( MakeDate(Literal(2019), Literal(7), Literal(32)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "DAY", "range" -> "1 ... 28/31", - "badValue" -> "32") + "badValue" -> "'32'") ) } @@ -1220,57 +1220,57 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { if (ansi) { checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(year = Literal(Int.MaxValue)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "YEAR", "range" -> "-999999999 ... 999999999", - "badValue" -> "2147483647") + "badValue" -> "'2147483647'") ) checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(month = Literal(13)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "MONTH", "range" -> "1 ... 12", - "badValue" -> "13") + "badValue" -> "'13'") ) checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(day = Literal(32)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "DAY", "range" -> "1 ... 28/31", - "badValue" -> "32") + "badValue" -> "'32'") ) checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(hour = Literal(25)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "HOUR", "range" -> "0 ... 23", - "badValue" -> "25") + "badValue" -> "'25'") ) checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(min = Literal(65)), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "MINUTE", "range" -> "0 ... 59", - "badValue" -> "65") + "badValue" -> "'65'") ) checkErrorInExpression[SparkDateTimeException]( makeTimestampExpr.copy(sec = Literal(Decimal(BigDecimal(70.0), 16, 6))), - "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "DATETIME_FIELD_OUT_OF_BOUNDS", Map( "ansiConfig" -> "\"spark.sql.ansi.enabled\"", "unit" -> "SECOND", "range" -> "0 ... 59", - "badValue" -> "70") + "badValue" -> "'70'") ) } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out index 0982aeec873c..5fd834e5808d 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out @@ -53,11 +53,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "13", + "badValue" : "'13'", "range" : "1 ... 12", "unit" : "MONTH" } @@ -71,11 +71,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "33", + "badValue" : "'33'", "range" : "1 ... 28/31", "unit" : "DAY" } diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out index 0456df781934..24f235fdbaed 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/timestamp.sql.out @@ -154,11 +154,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "61", + "badValue" : "'61'", "range" : "0 ... 59", "unit" : "SECOND" } @@ -188,11 +188,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "99", + "badValue" : "'99'", "range" : "0 ... 59", "unit" : "SECOND" } @@ -206,11 +206,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "999", + "badValue" : "'999'", "range" : "0 ... 59", "unit" : "SECOND" } diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 1af862d6aab2..a53336859f88 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -687,11 +687,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "30", + "badValue" : "'30'", "range" : "1 ... 28/31", "unit" : "DAY" } @@ -705,11 +705,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "13", + "badValue" : "'13'", "range" : "1 ... 12", "unit" : "MONTH" } @@ -723,11 +723,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "-1", + "badValue" : "'-1'", "range" : "1 ... 28/31", "unit" : "DAY" } diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out index 2ebbc33d5bb2..dc7624b37a3a 100644 --- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out @@ -154,11 +154,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "61", + "badValue" : "'61'", "range" : "0 ... 59", "unit" : "SECOND" } @@ -188,11 +188,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "99", + "badValue" : "'99'", "range" : "0 ... 59", "unit" : "SECOND" } @@ -206,11 +206,11 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATE_TIME_FIELD_OUT_OF_BOUNDS", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS", "sqlState" : "22008", "messageParameters" : { "ansiConfig" : "\"spark.sql.ansi.enabled\"", - "badValue" : "999", + "badValue" : "'999'", "range" : "0 ... 59", "unit" : "SECOND" } From 7a31955c5a91a602ebd074d31c4fd98f96acc7ee Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Thu, 3 Oct 2024 23:24:44 +0900 Subject: [PATCH 13/14] revert ansiIllegalArgumentError --- .../src/main/resources/error/error-conditions.json | 5 +++++ .../spark/sql/catalyst/util/DateTimeUtils.scala | 3 ++- .../spark/sql/errors/QueryExecutionErrors.scala | 11 ++++++----- .../catalyst/expressions/DateExpressionsSuite.scala | 12 +++++------- .../spark/sql/catalyst/util/DateTimeUtilsSuite.scala | 12 +++++------- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 68aac83f2d29..ebaa2cee7c8d 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -6707,6 +6707,11 @@ "Sinks cannot request distribution and ordering in continuous execution mode." ] }, + "_LEGACY_ERROR_TEMP_2000" : { + "message" : [ + ". If necessary set to false to bypass this error." + ] + }, "_LEGACY_ERROR_TEMP_2003" : { "message" : [ "Unsuccessful try to zip maps with unique keys due to exceeding the array size limit ." diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 8c20d0bc252a..e27ce29fc231 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -304,7 +304,8 @@ object DateTimeUtils extends SparkDateTimeUtils { start: Int, interval: CalendarInterval): Int = { if (interval.microseconds != 0) { - throw QueryExecutionErrors.ansiIllegalArgumentError() + throw QueryExecutionErrors.ansiIllegalArgumentError( + "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date") } val ld = daysToLocalDate(start).plusMonths(interval.months).plusDays(interval.days) localDateToDays(ld) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index 4e62a8427573..cc9868443b8a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -315,11 +315,12 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE ) } - def ansiIllegalArgumentError(): SparkException = { - SparkException.internalError( - "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + - s"If necessary set ${toSQLConf(SQLConf.ANSI_ENABLED.key)} to false to bypass this error" - ) + def ansiIllegalArgumentError(message: String): SparkIllegalArgumentException = { + new SparkIllegalArgumentException( + errorClass = "_LEGACY_ERROR_TEMP_2000", + messageParameters = Map( + "message" -> message, + "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key))) } def overflowInSumOfDecimalError(context: QueryContext): ArithmeticException = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 4a7135151a74..35d1eeee9442 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -434,14 +434,12 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } withSQLConf((SQLConf.ANSI_ENABLED.key, "true")) { - checkErrorInExpression[SparkException]( + checkErrorInExpression[SparkIllegalArgumentException]( DateAddInterval(Literal(d), Literal(new CalendarInterval(1, 1, 25 * MICROS_PER_HOUR))), - condition = "INTERNAL_ERROR", - parameters = Map( - "message" -> - ("Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + - "If necessary set \"spark.sql.ansi.enabled\" to false to bypass this error")) - ) + "_LEGACY_ERROR_TEMP_2000", + Map("message" -> + "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", + "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala index d2bdfe0c2499..96aaf13052b0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit import org.scalatest.matchers.must.Matchers import org.scalatest.matchers.should.Matchers._ -import org.apache.spark.{SparkException, SparkFunSuite, SparkIllegalArgumentException} +import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException} import org.apache.spark.sql.catalyst.plans.SQLHelper import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ @@ -540,14 +540,12 @@ class DateTimeUtilsSuite extends SparkFunSuite with Matchers with SQLHelper { assert(dateAddInterval(input, new CalendarInterval(36, 47, 0)) === days(2000, 4, 15)) assert(dateAddInterval(input, new CalendarInterval(-13, 0, 0)) === days(1996, 1, 28)) checkError( - exception = intercept[SparkException]( + exception = intercept[SparkIllegalArgumentException]( dateAddInterval(input, new CalendarInterval(36, 47, 1))), - condition = "INTERNAL_ERROR", + condition = "_LEGACY_ERROR_TEMP_2000", parameters = Map( - "message" -> - ("Cannot add hours, minutes or seconds, milliseconds, microseconds to a date. " + - "If necessary set \"spark.sql.ansi.enabled\" to false to bypass this error")) - ) + "message" -> "Cannot add hours, minutes or seconds, milliseconds, microseconds to a date", + "ansiConfig" -> "\"spark.sql.ansi.enabled\"")) } test("timestamp add interval") { From 6d96a85b1b6944f4f6dc143092a0dde821417c13 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Sat, 5 Oct 2024 16:52:16 +0900 Subject: [PATCH 14/14] raise existing error for default cases --- .../apache/spark/sql/errors/QueryExecutionErrors.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala index cc9868443b8a..eeb9a2adaed8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala @@ -298,6 +298,14 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE (unit, formattedRange, badValue) case datePattern(badDate) => ("DAY", "1 ... 28/31", badDate) + case _ => + throw new SparkDateTimeException( + errorClass = "_LEGACY_ERROR_TEMP_2000", + messageParameters = Map( + "message" -> errorMessage, + "ansiConfig" -> toSQLConf(SQLConf.ANSI_ENABLED.key)), + context = Array.empty, + summary = "") } } val (unit, range, badValue) = extractDateTimeErrorInfo(e)