Skip to content

Commit c2ed15d

Browse files
gengliangwanggengliangwang
committed
[SPARK-38710][SQL] Use SparkArithmeticException for arithmetic overflow runtime errors
### What changes were proposed in this pull request? On arithmetic overflow runtime errors, Spark should throw SparkArithmeticException instead of `java.lang.ArithmeticException` ### Why are the changes needed? Use a better error exception type. ### Does this PR introduce _any_ user-facing change? Yes, trivial change on the exception type: on arithmetic overflow runtime errors, Spark will throw SparkArithmeticException instead of `java.lang.ArithmeticException` ### How was this patch tested? UT Closes #36022 from gengliangwang/ArithmeticException. Authored-by: Gengliang Wang <[email protected]> Signed-off-by: Gengliang Wang <[email protected]> (cherry picked from commit b70fa24) Signed-off-by: Gengliang Wang <[email protected]>
1 parent eb44279 commit c2ed15d

File tree

8 files changed

+36
-33
lines changed

8 files changed

+36
-33
lines changed

‎core/src/main/resources/error/error-classes.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
"message" : [ "Field name %s is ambiguous and has %s matching fields in the struct." ],
44
"sqlState" : "42000"
55
},
6+
"ARITHMETIC_OVERFLOW" : {
7+
"message" : [ "%s.%s If necessary set %s to false (except for ANSI interval type) to bypass this error.%s" ],
8+
"sqlState" : "22003"
9+
},
610
"CANNOT_CAST_DATATYPE" : {
711
"message" : [ "Cannot cast %s to %s." ],
812
"sqlState" : "22005"

‎sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -446,9 +446,8 @@ object QueryExecutionErrors {
446446
hint: String = "",
447447
errorContext: String = ""): ArithmeticException = {
448448
val alternative = if (hint.nonEmpty) s" To return NULL instead, use '$hint'." else ""
449-
new ArithmeticException(s"$message.$alternative If necessary set " +
450-
s"${SQLConf.ANSI_ENABLED.key} to false (except for ANSI interval type) to bypass this " +
451-
"error." + errorContext)
449+
new SparkArithmeticException("ARITHMETIC_OVERFLOW",
450+
Array(message, alternative, SQLConf.ANSI_ENABLED.key, errorContext))
452451
}
453452

454453
def unaryMinusCauseOverflowError(originValue: AnyVal): ArithmeticException = {

‎sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1755,7 +1755,7 @@ select -(a) from values (interval '-2147483648 months', interval '2147483647 mon
17551755
-- !query schema
17561756
struct<>
17571757
-- !query output
1758-
java.lang.ArithmeticException
1758+
org.apache.spark.SparkArithmeticException
17591759
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17601760

17611761

@@ -1764,7 +1764,7 @@ select a - b from values (interval '-2147483648 months', interval '2147483647 mo
17641764
-- !query schema
17651765
struct<>
17661766
-- !query output
1767-
java.lang.ArithmeticException
1767+
org.apache.spark.SparkArithmeticException
17681768
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17691769

17701770

@@ -1773,7 +1773,7 @@ select b + interval '1 month' from values (interval '-2147483648 months', interv
17731773
-- !query schema
17741774
struct<>
17751775
-- !query output
1776-
java.lang.ArithmeticException
1776+
org.apache.spark.SparkArithmeticException
17771777
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17781778

17791779

@@ -2002,7 +2002,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
20022002
-- !query schema
20032003
struct<>
20042004
-- !query output
2005-
java.lang.ArithmeticException
2005+
org.apache.spark.SparkArithmeticException
20062006
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20072007
== SQL(line 1, position 7) ==
20082008
SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
@@ -2014,7 +2014,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
20142014
-- !query schema
20152015
struct<>
20162016
-- !query output
2017-
java.lang.ArithmeticException
2017+
org.apache.spark.SparkArithmeticException
20182018
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20192019
== SQL(line 1, position 7) ==
20202020
SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
@@ -2060,7 +2060,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
20602060
-- !query schema
20612061
struct<>
20622062
-- !query output
2063-
java.lang.ArithmeticException
2063+
org.apache.spark.SparkArithmeticException
20642064
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20652065
== SQL(line 1, position 7) ==
20662066
SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
@@ -2072,7 +2072,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L
20722072
-- !query schema
20732073
struct<>
20742074
-- !query output
2075-
java.lang.ArithmeticException
2075+
org.apache.spark.SparkArithmeticException
20762076
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20772077
== SQL(line 1, position 7) ==
20782078
SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L

‎sql/core/src/test/resources/sql-tests/results/interval.sql.out‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,7 +1744,7 @@ select -(a) from values (interval '-2147483648 months', interval '2147483647 mon
17441744
-- !query schema
17451745
struct<>
17461746
-- !query output
1747-
java.lang.ArithmeticException
1747+
org.apache.spark.SparkArithmeticException
17481748
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17491749

17501750

@@ -1753,7 +1753,7 @@ select a - b from values (interval '-2147483648 months', interval '2147483647 mo
17531753
-- !query schema
17541754
struct<>
17551755
-- !query output
1756-
java.lang.ArithmeticException
1756+
org.apache.spark.SparkArithmeticException
17571757
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17581758

17591759

@@ -1762,7 +1762,7 @@ select b + interval '1 month' from values (interval '-2147483648 months', interv
17621762
-- !query schema
17631763
struct<>
17641764
-- !query output
1765-
java.lang.ArithmeticException
1765+
org.apache.spark.SparkArithmeticException
17661766
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
17671767

17681768

@@ -1991,7 +1991,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
19911991
-- !query schema
19921992
struct<>
19931993
-- !query output
1994-
java.lang.ArithmeticException
1994+
org.apache.spark.SparkArithmeticException
19951995
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
19961996
== SQL(line 1, position 7) ==
19971997
SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1
@@ -2003,7 +2003,7 @@ SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
20032003
-- !query schema
20042004
struct<>
20052005
-- !query output
2006-
java.lang.ArithmeticException
2006+
org.apache.spark.SparkArithmeticException
20072007
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20082008
== SQL(line 1, position 7) ==
20092009
SELECT (INTERVAL '-178956970-8' YEAR TO MONTH) / -1L
@@ -2049,7 +2049,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
20492049
-- !query schema
20502050
struct<>
20512051
-- !query output
2052-
java.lang.ArithmeticException
2052+
org.apache.spark.SparkArithmeticException
20532053
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20542054
== SQL(line 1, position 7) ==
20552055
SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1
@@ -2061,7 +2061,7 @@ SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L
20612061
-- !query schema
20622062
struct<>
20632063
-- !query output
2064-
java.lang.ArithmeticException
2064+
org.apache.spark.SparkArithmeticException
20652065
Overflow in integral divide. To return NULL instead, use 'try_divide'. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
20662066
== SQL(line 1, position 7) ==
20672067
SELECT (INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND) / -1L

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/int4.sql.out‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ SELECT '' AS five, i.f1, i.f1 * smallint('2') AS x FROM INT4_TBL i
199199
-- !query schema
200200
struct<>
201201
-- !query output
202-
java.lang.ArithmeticException
202+
org.apache.spark.SparkArithmeticException
203203
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
204204
== SQL(line 1, position 25) ==
205205
SELECT '' AS five, i.f1, i.f1 * smallint('2') AS x FROM INT4_TBL i
@@ -222,7 +222,7 @@ SELECT '' AS five, i.f1, i.f1 * int('2') AS x FROM INT4_TBL i
222222
-- !query schema
223223
struct<>
224224
-- !query output
225-
java.lang.ArithmeticException
225+
org.apache.spark.SparkArithmeticException
226226
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
227227
== SQL(line 1, position 25) ==
228228
SELECT '' AS five, i.f1, i.f1 * int('2') AS x FROM INT4_TBL i
@@ -245,7 +245,7 @@ SELECT '' AS five, i.f1, i.f1 + smallint('2') AS x FROM INT4_TBL i
245245
-- !query schema
246246
struct<>
247247
-- !query output
248-
java.lang.ArithmeticException
248+
org.apache.spark.SparkArithmeticException
249249
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
250250
== SQL(line 1, position 25) ==
251251
SELECT '' AS five, i.f1, i.f1 + smallint('2') AS x FROM INT4_TBL i
@@ -269,7 +269,7 @@ SELECT '' AS five, i.f1, i.f1 + int('2') AS x FROM INT4_TBL i
269269
-- !query schema
270270
struct<>
271271
-- !query output
272-
java.lang.ArithmeticException
272+
org.apache.spark.SparkArithmeticException
273273
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
274274
== SQL(line 1, position 25) ==
275275
SELECT '' AS five, i.f1, i.f1 + int('2') AS x FROM INT4_TBL i
@@ -293,7 +293,7 @@ SELECT '' AS five, i.f1, i.f1 - smallint('2') AS x FROM INT4_TBL i
293293
-- !query schema
294294
struct<>
295295
-- !query output
296-
java.lang.ArithmeticException
296+
org.apache.spark.SparkArithmeticException
297297
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
298298
== SQL(line 1, position 25) ==
299299
SELECT '' AS five, i.f1, i.f1 - smallint('2') AS x FROM INT4_TBL i
@@ -317,7 +317,7 @@ SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i
317317
-- !query schema
318318
struct<>
319319
-- !query output
320-
java.lang.ArithmeticException
320+
org.apache.spark.SparkArithmeticException
321321
integer overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
322322
== SQL(line 1, position 25) ==
323323
SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT4_TBL i

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/int8.sql.out‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,7 @@ SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL
391391
-- !query schema
392392
struct<>
393393
-- !query output
394-
java.lang.ArithmeticException
394+
org.apache.spark.SparkArithmeticException
395395
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
396396
== SQL(line 1, position 28) ==
397397
SELECT '' AS three, q1, q2, q1 * q2 AS multiply FROM INT8_TBL
@@ -753,7 +753,7 @@ SELECT bigint((-9223372036854775808)) * bigint((-1))
753753
-- !query schema
754754
struct<>
755755
-- !query output
756-
java.lang.ArithmeticException
756+
org.apache.spark.SparkArithmeticException
757757
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
758758
== SQL(line 1, position 7) ==
759759
SELECT bigint((-9223372036854775808)) * bigint((-1))
@@ -781,7 +781,7 @@ SELECT bigint((-9223372036854775808)) * int((-1))
781781
-- !query schema
782782
struct<>
783783
-- !query output
784-
java.lang.ArithmeticException
784+
org.apache.spark.SparkArithmeticException
785785
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
786786
== SQL(line 1, position 7) ==
787787
SELECT bigint((-9223372036854775808)) * int((-1))
@@ -809,7 +809,7 @@ SELECT bigint((-9223372036854775808)) * smallint((-1))
809809
-- !query schema
810810
struct<>
811811
-- !query output
812-
java.lang.ArithmeticException
812+
org.apache.spark.SparkArithmeticException
813813
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
814814
== SQL(line 1, position 7) ==
815815
SELECT bigint((-9223372036854775808)) * smallint((-1))

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/window_part2.sql.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ from range(9223372036854775804, 9223372036854775807) x
224224
-- !query schema
225225
struct<>
226226
-- !query output
227-
java.lang.ArithmeticException
227+
org.apache.spark.SparkArithmeticException
228228
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
229229

230230

@@ -234,7 +234,7 @@ from range(-9223372036854775806, -9223372036854775805) x
234234
-- !query schema
235235
struct<>
236236
-- !query output
237-
java.lang.ArithmeticException
237+
org.apache.spark.SparkArithmeticException
238238
long overflow. If necessary set spark.sql.ansi.enabled to false (except for ANSI interval type) to bypass this error.
239239

240240

‎sql/core/src/test/scala/org/apache/spark/sql/DataFrameAggregateSuite.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1278,12 +1278,12 @@ class DataFrameAggregateSuite extends QueryTest
12781278
val error = intercept[SparkException] {
12791279
checkAnswer(df2.select(sum($"year-month")), Nil)
12801280
}
1281-
assert(error.toString contains "java.lang.ArithmeticException: integer overflow")
1281+
assert(error.toString contains "SparkArithmeticException: integer overflow")
12821282

12831283
val error2 = intercept[SparkException] {
12841284
checkAnswer(df2.select(sum($"day")), Nil)
12851285
}
1286-
assert(error2.toString contains "java.lang.ArithmeticException: long overflow")
1286+
assert(error2.toString contains "SparkArithmeticException: long overflow")
12871287
}
12881288

12891289
test("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") {
@@ -1412,12 +1412,12 @@ class DataFrameAggregateSuite extends QueryTest
14121412
val error = intercept[SparkException] {
14131413
checkAnswer(df2.select(avg($"year-month")), Nil)
14141414
}
1415-
assert(error.toString contains "java.lang.ArithmeticException: integer overflow")
1415+
assert(error.toString contains "SparkArithmeticException: integer overflow")
14161416

14171417
val error2 = intercept[SparkException] {
14181418
checkAnswer(df2.select(avg($"day")), Nil)
14191419
}
1420-
assert(error2.toString contains "java.lang.ArithmeticException: long overflow")
1420+
assert(error2.toString contains "SparkArithmeticException: long overflow")
14211421

14221422
val df3 = intervalData.filter($"class" > 4)
14231423
val avgDF3 = df3.select(avg($"year-month"), avg($"day"))

0 commit comments

Comments
 (0)