Skip to content

[SPARK-28343][FOLLOW-UP][SQL][TEST] Enable spark.sql.function.preferIntegralDivision for PostgreSQL testing #25170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-28343][FOLLOW-UP][SQL][TEST] Enable spark.sql.function.preferIntegralDivision for PostgreSQL testing #25170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions sql/core/src/test/resources/sql-tests/inputs/pgSQL/int2.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,9 @@ WHERE f1 > -32767;

SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT2_TBL i;

-- PostgreSQL `/` is the same with Spark `div` since SPARK-2659.
SELECT '' AS five, i.f1, i.f1 div smallint('2') AS x FROM INT2_TBL i;
SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT2_TBL i;

-- PostgreSQL `/` is the same with Spark `div` since SPARK-2659.
SELECT '' AS five, i.f1, i.f1 div int('2') AS x FROM INT2_TBL i;
SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT2_TBL i;

-- corner cases
SELECT string(shiftleft(smallint(-1), 15));
Expand Down
1 change: 0 additions & 1 deletion sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@ SELECT int('1000') < int('999') AS `false`;

SELECT 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 AS ten;

-- [SPARK-2659] HiveQL: Division operator should always perform fractional division
SELECT 2 + 2 / 2 AS three;

SELECT (2 + 2) / 2 AS two;
Expand Down
1 change: 0 additions & 1 deletion sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ SELECT 37 - q1 AS minus4 FROM INT8_TBL;
SELECT '' AS five, 2 * q1 AS `twice int4` FROM INT8_TBL;
SELECT '' AS five, q1 * 2 AS `twice int4` FROM INT8_TBL;

-- [SPARK-2659] HiveQL: Division operator should always perform fractional division
-- int8 op int4
SELECT q1 + int(42) AS `8plus4`, q1 - int(42) AS `8minus4`, q1 * int(42) AS `8mul4`, q1 / int(42) AS `8div4` FROM INT8_TBL;
-- int4 op int8
Expand Down
18 changes: 9 additions & 9 deletions sql/core/src/test/resources/sql-tests/results/pgSQL/case.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,28 +176,28 @@ struct<None:string,NULL on no matches:int>
-- !query 18
SELECT CASE WHEN 1=0 THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
-- !query 18 schema
struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
struct<CASE WHEN (1 = 0) THEN (1 div 0) WHEN (1 = 1) THEN 1 ELSE (2 div 0) END:int>
-- !query 18 output
1.0
1


-- !query 19
SELECT CASE 1 WHEN 0 THEN 1/0 WHEN 1 THEN 1 ELSE 2/0 END
-- !query 19 schema
struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
struct<CASE WHEN (1 = 0) THEN (1 div 0) WHEN (1 = 1) THEN 1 ELSE (2 div 0) END:int>
-- !query 19 output
1.0
1


-- !query 20
SELECT CASE WHEN i > 100 THEN 1/0 ELSE 0 END FROM case_tbl
-- !query 20 schema
struct<CASE WHEN (i > 100) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) ELSE CAST(0 AS DOUBLE) END:double>
struct<CASE WHEN (i > 100) THEN (1 div 0) ELSE 0 END:int>
-- !query 20 output
0.0
0.0
0.0
0.0
0
0
0
0


-- !query 21
Expand Down
4 changes: 2 additions & 2 deletions sql/core/src/test/resources/sql-tests/results/pgSQL/int2.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ struct<five:string,f1:smallint,x:int>


-- !query 27
SELECT '' AS five, i.f1, i.f1 div smallint('2') AS x FROM INT2_TBL i
SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT2_TBL i
-- !query 27 schema
struct<five:string,f1:smallint,x:smallint>
-- !query 27 output
Expand All @@ -278,7 +278,7 @@ struct<five:string,f1:smallint,x:smallint>


-- !query 28
SELECT '' AS five, i.f1, i.f1 div int('2') AS x FROM INT2_TBL i
SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT2_TBL i
-- !query 28 schema
struct<five:string,f1:smallint,x:int>
-- !query 28 output
Expand Down
32 changes: 16 additions & 16 deletions sql/core/src/test/resources/sql-tests/results/pgSQL/int4.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,25 +339,25 @@ struct<five:string,f1:int,x:int>
-- !query 33
SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT4_TBL i
-- !query 33 schema
struct<five:string,f1:int,x:double>
struct<five:string,f1:int,x:int>
-- !query 33 output
-123456 -61728.0
-2147483647 -1.0737418235E9
0 0.0
123456 61728.0
2147483647 1.0737418235E9
-123456 -61728
-2147483647 -1073741823
0 0
123456 61728
2147483647 1073741823


-- !query 34
SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT4_TBL i
-- !query 34 schema
struct<five:string,f1:int,x:double>
struct<five:string,f1:int,x:int>
-- !query 34 output
-123456 -61728.0
-2147483647 -1.0737418235E9
0 0.0
123456 61728.0
2147483647 1.0737418235E9
-123456 -61728
-2147483647 -1073741823
0 0
123456 61728
2147483647 1073741823


-- !query 35
Expand Down Expand Up @@ -435,17 +435,17 @@ struct<ten:int>
-- !query 44
SELECT 2 + 2 / 2 AS three
-- !query 44 schema
struct<three:double>
struct<three:int>
-- !query 44 output
3.0
3


-- !query 45
SELECT (2 + 2) / 2 AS two
-- !query 45 schema
struct<two:double>
struct<two:int>
-- !query 45 output
2.0
2


-- !query 46
Expand Down
78 changes: 39 additions & 39 deletions sql/core/src/test/resources/sql-tests/results/pgSQL/int8.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,13 +412,13 @@ struct<three:string,q1:bigint,q2:bigint,multiply:bigint>
-- !query 42
SELECT '' AS five, q1, q2, q1 / q2 AS divide, q1 % q2 AS mod FROM INT8_TBL
-- !query 42 schema
struct<five:string,q1:bigint,q2:bigint,divide:double,mod:bigint>
struct<five:string,q1:bigint,q2:bigint,divide:bigint,mod:bigint>
-- !query 42 output
123 456 0.26973684210526316 123
123 4567890123456789 2.6927092525360204E-14 123
4567890123456789 -4567890123456789 -1.0 0
4567890123456789 123 3.713731807688446E13 57
4567890123456789 4567890123456789 1.0 0
123 456 0 123
123 4567890123456789 0 123
4567890123456789 -4567890123456789 -1 0
4567890123456789 123 37137318076884 57
4567890123456789 4567890123456789 1 0


-- !query 43
Expand Down Expand Up @@ -496,49 +496,49 @@ struct<five:string,twice int4:bigint>
-- !query 49
SELECT q1 + int(42) AS `8plus4`, q1 - int(42) AS `8minus4`, q1 * int(42) AS `8mul4`, q1 / int(42) AS `8div4` FROM INT8_TBL
-- !query 49 schema
struct<8plus4:bigint,8minus4:bigint,8mul4:bigint,8div4:double>
struct<8plus4:bigint,8minus4:bigint,8mul4:bigint,8div4:bigint>
-- !query 49 output
165 81 5166 2.9285714285714284
165 81 5166 2.9285714285714284
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
165 81 5166 2
165 81 5166 2
4567890123456831 4567890123456747 191851385185185138 108759288653733
4567890123456831 4567890123456747 191851385185185138 108759288653733
4567890123456831 4567890123456747 191851385185185138 108759288653733


-- !query 50
SELECT int(246) + q1 AS `4plus8`, int(246) - q1 AS `4minus8`, int(246) * q1 AS `4mul8`, int(246) / q1 AS `4div8` FROM INT8_TBL
-- !query 50 schema
struct<4plus8:bigint,4minus8:bigint,4mul8:bigint,4div8:double>
struct<4plus8:bigint,4minus8:bigint,4mul8:bigint,4div8:bigint>
-- !query 50 output
369 123 30258 2.0
369 123 30258 2.0
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
369 123 30258 2
369 123 30258 2
4567890123457035 -4567890123456543 1123700970370370094 0
4567890123457035 -4567890123456543 1123700970370370094 0
4567890123457035 -4567890123456543 1123700970370370094 0


-- !query 51
SELECT q1 + smallint(42) AS `8plus2`, q1 - smallint(42) AS `8minus2`, q1 * smallint(42) AS `8mul2`, q1 / smallint(42) AS `8div2` FROM INT8_TBL
-- !query 51 schema
struct<8plus2:bigint,8minus2:bigint,8mul2:bigint,8div2:double>
struct<8plus2:bigint,8minus2:bigint,8mul2:bigint,8div2:bigint>
-- !query 51 output
165 81 5166 2.9285714285714284
165 81 5166 2.9285714285714284
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
4567890123456831 4567890123456747 191851385185185138 1.0875928865373308E14
165 81 5166 2
165 81 5166 2
4567890123456831 4567890123456747 191851385185185138 108759288653733
4567890123456831 4567890123456747 191851385185185138 108759288653733
4567890123456831 4567890123456747 191851385185185138 108759288653733


-- !query 52
SELECT smallint(246) + q1 AS `2plus8`, smallint(246) - q1 AS `2minus8`, smallint(246) * q1 AS `2mul8`, smallint(246) / q1 AS `2div8` FROM INT8_TBL
-- !query 52 schema
struct<2plus8:bigint,2minus8:bigint,2mul8:bigint,2div8:double>
struct<2plus8:bigint,2minus8:bigint,2mul8:bigint,2div8:bigint>
-- !query 52 output
369 123 30258 2.0
369 123 30258 2.0
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
4567890123457035 -4567890123456543 1123700970370370094 5.385418505072041E-14
369 123 30258 2
369 123 30258 2
4567890123457035 -4567890123456543 1123700970370370094 0
4567890123457035 -4567890123456543 1123700970370370094 0
4567890123457035 -4567890123456543 1123700970370370094 0


-- !query 53
Expand Down Expand Up @@ -572,23 +572,23 @@ struct<max(q1):bigint,max(q2):bigint>
-- !query 56
select bigint('9223372036854775800') / bigint('0')
-- !query 56 schema
struct<(CAST(CAST(9223372036854775800 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS BIGINT) AS DOUBLE)):double>
struct<(CAST(9223372036854775800 AS BIGINT) div CAST(0 AS BIGINT)):bigint>
-- !query 56 output
NULL


-- !query 57
select bigint('-9223372036854775808') / smallint('0')
-- !query 57 schema
struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS SMALLINT) AS DOUBLE)):double>
struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(0 AS SMALLINT) AS BIGINT)):bigint>
-- !query 57 output
NULL


-- !query 58
select smallint('100') / bigint('0')
-- !query 58 schema
struct<(CAST(CAST(100 AS SMALLINT) AS DOUBLE) / CAST(CAST(0 AS BIGINT) AS DOUBLE)):double>
struct<(CAST(CAST(100 AS SMALLINT) AS BIGINT) div CAST(0 AS BIGINT)):bigint>
-- !query 58 output
NULL

Expand Down Expand Up @@ -744,9 +744,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(-1 AS BIGINT)):bigint>
-- !query 74
SELECT bigint((-9223372036854775808)) / bigint((-1))
-- !query 74 schema
struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS BIGINT) AS DOUBLE)):double>
struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(-1 AS BIGINT)):bigint>
-- !query 74 output
9.223372036854776E18
-9223372036854775808


-- !query 75
Expand All @@ -768,9 +768,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(CAST(-1 AS INT) AS BIGINT)):
-- !query 77
SELECT bigint((-9223372036854775808)) / int((-1))
-- !query 77 schema
struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS INT) AS DOUBLE)):double>
struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(-1 AS INT) AS BIGINT)):bigint>
-- !query 77 output
9.223372036854776E18
-9223372036854775808


-- !query 78
Expand All @@ -792,9 +792,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(CAST(-1 AS SMALLINT) AS BIGI
-- !query 80
SELECT bigint((-9223372036854775808)) / smallint((-1))
-- !query 80 schema
struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS SMALLINT) AS DOUBLE)):double>
struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(-1 AS SMALLINT) AS BIGINT)):bigint>
-- !query 80 output
9.223372036854776E18
-9223372036854775808


-- !query 81
Expand Down
8 changes: 4 additions & 4 deletions sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-case.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,23 +176,23 @@ struct<None:string,NULL on no matches:int>
-- !query 18
SELECT CASE WHEN CAST(udf(1=0) AS boolean) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
-- !query 18 schema
struct<CASE WHEN CAST(udf((1 = 0)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
struct<CASE WHEN CAST(udf((1 = 0)) AS BOOLEAN) THEN (1 div 0) WHEN (1 = 1) THEN 1 ELSE (2 div 0) END:int>
-- !query 18 output
1.0
1


-- !query 19
SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END
-- !query 19 schema
struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(0) AS DOUBLE) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE)) END:double>
struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(0) AS DOUBLE) AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE CAST((2 div 0) AS DOUBLE) END:double>
-- !query 19 output
1.0


-- !query 20
SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl
-- !query 20 schema
struct<CASE WHEN (i > 100) THEN udf((cast(1 as double) / cast(0 as double))) ELSE udf(0) END:string>
struct<CASE WHEN (i > 100) THEN udf((1 div 0)) ELSE udf(0) END:string>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you @wangyum for updating this file together.

-- !query 20 output
0
0
Expand Down
1 change: 1 addition & 0 deletions sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
// PostgreSQL enabled cartesian product by default.
localSparkSession.conf.set(SQLConf.CROSS_JOINS_ENABLED.key, true)
localSparkSession.conf.set(SQLConf.ANSI_SQL_PARSER.key, true)
localSparkSession.conf.set(SQLConf.PREFER_INTEGRAL_DIVISION.key, true)
case _ =>
}

Expand Down