Skip to content

Commit 403ef53

Browse files
HyukjinKwonHyukjinKwon
committed
Revert "Revert "Revert "[SPARK-28461][SQL] Pad Decimal numbers with trailing zeros to the scale of the column"""
This reverts commit 54a233c.
1 parent 54a233c commit 403ef53

36 files changed

+807
-847
lines changed

‎docs/sql-migration-guide.md‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -226,32 +226,6 @@ license: |
226226

227227
- Since Spark 3.0, when casting string value to date, timestamp and interval values, the leading and trailing white spaces(<= ACSII 32) will be trimmed before casing, e.g. `cast('2019-10-10\t as date)` results the date value `2019-10-10`. In Spark version 2.4 and earlier, only the trailing space will be removed, thus, the result is `null`.
228228

229-
- Since Spark 3.0, we pad decimal numbers with trailing zeros to the scale of the column for `spark-sql` interface, for example:
230-
<table class="table">
231-
<tr>
232-
<th>
233-
<b>Query</b>
234-
</th>
235-
<th>
236-
<b>Spark 2.4 or Prior</b>
237-
</th>
238-
<th>
239-
<b>Spark 3.0</b>
240-
</th>
241-
</tr>
242-
<tr>
243-
<td>
244-
<code>SELECT CAST(1 AS decimal(38, 18));</code>
245-
</td>
246-
<td>
247-
<code>1</code>
248-
</td>
249-
<td>
250-
<code>1.000000000000000000</code>
251-
</td>
252-
</tr>
253-
</table>
254-
255229
## Upgrading from Spark SQL 2.4 to 2.4.1
256230

257231
- The value of `spark.executor.heartbeatInterval`, when specified without units like "30" rather than "30s", was

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ import org.apache.spark.sql.types._
6363
> SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100);
6464
[10.0,10.0,10.0]
6565
> SELECT _FUNC_(10.0, 0.5, 100);
66-
10.0
66+
10
6767
""",
6868
since = "2.1.0")
6969
case class ApproximatePercentile(

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,9 +111,9 @@ case class If(predicate: Expression, trueValue: Expression, falseValue: Expressi
111111
examples = """
112112
Examples:
113113
> SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
114-
1.0
114+
1
115115
> SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;
116-
2.0
116+
2
117117
> SELECT CASE WHEN 1 < 0 THEN 1 WHEN 2 < 0 THEN 2.0 END;
118118
NULL
119119
""")

‎sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ object HiveResult {
5959
result.map(_.zip(types).map(toHiveString)).map(_.mkString("\t"))
6060
}
6161

62+
private def formatDecimal(d: java.math.BigDecimal): String = {
63+
if (d.compareTo(java.math.BigDecimal.ZERO) == 0) {
64+
java.math.BigDecimal.ZERO.toPlainString
65+
} else {
66+
d.stripTrailingZeros().toPlainString // Hive strips trailing zeros
67+
}
68+
}
69+
6270
private val primitiveTypes = Seq(
6371
StringType,
6472
IntegerType,
@@ -119,7 +127,7 @@ object HiveResult {
119127
case (t: Timestamp, TimestampType) =>
120128
DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t))
121129
case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
122-
case (decimal: java.math.BigDecimal, DecimalType()) => decimal.toPlainString
130+
case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal)
123131
case (interval: CalendarInterval, CalendarIntervalType) =>
124132
SQLConf.get.intervalOutputStyle match {
125133
case SQL_STANDARD => toSqlStandardString(interval)

‎sql/core/src/test/resources/sql-tests/results/ansi/decimalArithmeticOperations.sql.out‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,50 +24,50 @@ select id, a*10, b/10 from decimals_test order by id
2424
-- !query 2 schema
2525
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
2626
-- !query 2 output
27-
1 1000.000000000000000 99.900000000000000000
28-
2 123451.230000000000000 1234.512300000000000000
29-
3 1.234567891011000 123.410000000000000000
30-
4 1234567891234567890.000000000000000 0.112345678912345679
27+
1 1000 99.9
28+
2 123451.23 1234.5123
29+
3 1.234567891011 123.41
30+
4 1234567891234567890 0.112345678912345679
3131

3232

3333
-- !query 3
3434
select 10.3 * 3.0
3535
-- !query 3 schema
3636
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
3737
-- !query 3 output
38-
30.90
38+
30.9
3939

4040

4141
-- !query 4
4242
select 10.3000 * 3.0
4343
-- !query 4 schema
4444
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
4545
-- !query 4 output
46-
30.90000
46+
30.9
4747

4848

4949
-- !query 5
5050
select 10.30000 * 30.0
5151
-- !query 5 schema
5252
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
5353
-- !query 5 output
54-
309.000000
54+
309
5555

5656

5757
-- !query 6
5858
select 10.300000000000000000 * 3.000000000000000000
5959
-- !query 6 schema
6060
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
6161
-- !query 6 output
62-
30.9000000000000000000000000000000000
62+
30.9
6363

6464

6565
-- !query 7
6666
select 10.300000000000000000 * 3.0000000000000000000
6767
-- !query 7 schema
6868
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
6969
-- !query 7 output
70-
30.9000000000000000000000000000000000
70+
30.9
7171

7272

7373
-- !query 8

‎sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
169169
-- !query 17 schema
170170
struct<1.0:double,1.2:double,1E+10:decimal(1,-10),1.5E+5:decimal(2,-4),0.1:double,0.1:double,1E+4:decimal(1,-4),9E+1:decimal(1,-1),9E+1:decimal(1,-1),90.0:decimal(3,1),9E+1:decimal(1,-1)>
171171
-- !query 17 output
172-
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90.0 90
172+
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90 90
173173

174174

175175
-- !query 18
@@ -339,7 +339,7 @@ select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
339339
-- !query 35 schema
340340
struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
341341
-- !query 35 output
342-
90912830918230182310293801923652346786 0.00000000000000000000000001230 123.08
342+
90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
343343

344344

345345
-- !query 36

‎sql/core/src/test/resources/sql-tests/results/decimalArithmeticOperations.sql.out‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ select id, a+b, a-b, a*b, a/b from decimals_test order by id
5656
-- !query 6 schema
5757
struct<id:int,(a + b):decimal(38,17),(a - b):decimal(38,17),(a * b):decimal(38,6),(a / b):decimal(38,6)>
5858
-- !query 6 output
59-
1 1099.00000000000000000 -899.00000000000000000 99900.000000 0.100100
60-
2 24690.24600000000000000 0.00000000000000000 152402061.885129 1.000000
61-
3 1234.22345678910110000 -1233.97654321089890000 152.358023 0.000100
59+
1 1099 -899 99900 0.1001
60+
2 24690.246 0 152402061.885129 1
61+
3 1234.2234567891011 -1233.9765432108989 152.358023 0.0001
6262
4 123456789123456790.12345678912345679 123456789123456787.87654321087654321 138698367904130467.515623 109890109097814272.043109
6363

6464

@@ -67,50 +67,50 @@ select id, a*10, b/10 from decimals_test order by id
6767
-- !query 7 schema
6868
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,15),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18)>
6969
-- !query 7 output
70-
1 1000.000000000000000 99.900000000000000000
71-
2 123451.230000000000000 1234.512300000000000000
72-
3 1.234567891011000 123.410000000000000000
73-
4 1234567891234567890.000000000000000 0.112345678912345679
70+
1 1000 99.9
71+
2 123451.23 1234.5123
72+
3 1.234567891011 123.41
73+
4 1234567891234567890 0.112345678912345679
7474

7575

7676
-- !query 8
7777
select 10.3 * 3.0
7878
-- !query 8 schema
7979
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
8080
-- !query 8 output
81-
30.90
81+
30.9
8282

8383

8484
-- !query 9
8585
select 10.3000 * 3.0
8686
-- !query 9 schema
8787
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
8888
-- !query 9 output
89-
30.90000
89+
30.9
9090

9191

9292
-- !query 10
9393
select 10.30000 * 30.0
9494
-- !query 10 schema
9595
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
9696
-- !query 10 output
97-
309.000000
97+
309
9898

9999

100100
-- !query 11
101101
select 10.300000000000000000 * 3.000000000000000000
102102
-- !query 11 schema
103103
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,34)>
104104
-- !query 11 output
105-
30.9000000000000000000000000000000000
105+
30.9
106106

107107

108108
-- !query 12
109109
select 10.300000000000000000 * 3.0000000000000000000
110110
-- !query 12 schema
111111
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,34)>
112112
-- !query 12 output
113-
30.9000000000000000000000000000000000
113+
30.9
114114

115115

116116
-- !query 13
@@ -198,9 +198,9 @@ select id, a+b, a-b, a*b, a/b from decimals_test order by id
198198
-- !query 23 schema
199199
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
200200
-- !query 23 output
201-
1 1099.000000000000000000 -899.000000000000000000 NULL 0.100100100100100100
202-
2 24690.246000000000000000 0.000000000000000000 NULL 1.000000000000000000
203-
3 1234.223456789101100000 -1233.976543210898900000 NULL 0.000100037913541123
201+
1 1099 -899 NULL 0.1001001001001001
202+
2 24690.246 0 NULL 1
203+
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
204204
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436
205205

206206

@@ -209,42 +209,42 @@ select id, a*10, b/10 from decimals_test order by id
209209
-- !query 24 schema
210210
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
211211
-- !query 24 output
212-
1 1000.000000000000000000 99.9000000000000000000
213-
2 123451.230000000000000000 1234.5123000000000000000
214-
3 1.234567891011000000 123.4100000000000000000
215-
4 1234567891234567890.000000000000000000 0.1123456789123456789
212+
1 1000 99.9
213+
2 123451.23 1234.5123
214+
3 1.234567891011 123.41
215+
4 1234567891234567890 0.1123456789123456789
216216

217217

218218
-- !query 25
219219
select 10.3 * 3.0
220220
-- !query 25 schema
221221
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
222222
-- !query 25 output
223-
30.90
223+
30.9
224224

225225

226226
-- !query 26
227227
select 10.3000 * 3.0
228228
-- !query 26 schema
229229
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
230230
-- !query 26 output
231-
30.90000
231+
30.9
232232

233233

234234
-- !query 27
235235
select 10.30000 * 30.0
236236
-- !query 27 schema
237237
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
238238
-- !query 27 output
239-
309.000000
239+
309
240240

241241

242242
-- !query 28
243243
select 10.300000000000000000 * 3.000000000000000000
244244
-- !query 28 schema
245245
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
246246
-- !query 28 output
247-
30.900000000000000000000000000000000000
247+
30.9
248248

249249

250250
-- !query 29

‎sql/core/src/test/resources/sql-tests/results/literals.sql.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ select 1D, 1.2D, 1e10, 1.5e5, .10D, 0.10D, .1e5, .9e+2, 0.9e+2, 900e-1, 9.e+1
169169
-- !query 17 schema
170170
struct<1.0:double,1.2:double,1E+10:decimal(1,-10),1.5E+5:decimal(2,-4),0.1:double,0.1:double,1E+4:decimal(1,-4),9E+1:decimal(1,-1),9E+1:decimal(1,-1),90.0:decimal(3,1),9E+1:decimal(1,-1)>
171171
-- !query 17 output
172-
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90.0 90
172+
1.0 1.2 10000000000 150000 0.1 0.1 10000 90 90 90 90
173173

174174

175175
-- !query 18
@@ -339,7 +339,7 @@ select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
339339
-- !query 35 schema
340340
struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
341341
-- !query 35 output
342-
90912830918230182310293801923652346786 0.00000000000000000000000001230 123.08
342+
90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
343343

344344

345345
-- !query 36

‎sql/core/src/test/resources/sql-tests/results/order-by-nulls-ordering.sql.out‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -206,44 +206,44 @@ select * from spark_10747_mix order by col1 nulls last, col5 nulls last
206206
-- !query 13 schema
207207
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
208208
-- !query 13 output
209-
a 1 1.0 1.00 NULL
210-
b 2 1.0 1.00 10.0
211-
c 3 2.0 2.00 15.1
212-
c 3 2.0 2.00 NULL
213-
d 3 2.0 3.00 0.0
214-
d 3 0.0 3.00 1.0
215-
d 3 NULL 4.00 1.0
216-
NULL 3 0.0 3.00 1.0
209+
a 1 1.0 1 NULL
210+
b 2 1.0 1 10
211+
c 3 2.0 2 15.1
212+
c 3 2.0 2 NULL
213+
d 3 2.0 3 0
214+
d 3 0.0 3 1
215+
d 3 NULL 4 1
216+
NULL 3 0.0 3 1
217217

218218

219219
-- !query 14
220220
select * from spark_10747_mix order by col1 desc nulls first, col5 desc nulls first
221221
-- !query 14 schema
222222
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
223223
-- !query 14 output
224-
NULL 3 0.0 3.00 1.0
225-
d 3 0.0 3.00 1.0
226-
d 3 NULL 4.00 1.0
227-
d 3 2.0 3.00 0.0
228-
c 3 2.0 2.00 NULL
229-
c 3 2.0 2.00 15.1
230-
b 2 1.0 1.00 10.0
231-
a 1 1.0 1.00 NULL
224+
NULL 3 0.0 3 1
225+
d 3 0.0 3 1
226+
d 3 NULL 4 1
227+
d 3 2.0 3 0
228+
c 3 2.0 2 NULL
229+
c 3 2.0 2 15.1
230+
b 2 1.0 1 10
231+
a 1 1.0 1 NULL
232232

233233

234234
-- !query 15
235235
select * from spark_10747_mix order by col5 desc nulls first, col3 desc nulls last
236236
-- !query 15 schema
237237
struct<col1:string,col2:int,col3:double,col4:decimal(10,2),col5:decimal(20,1)>
238238
-- !query 15 output
239-
c 3 2.0 2.00 NULL
240-
a 1 1.0 1.00 NULL
241-
c 3 2.0 2.00 15.1
242-
b 2 1.0 1.00 10.0
243-
d 3 0.0 3.00 1.0
244-
NULL 3 0.0 3.00 1.0
245-
d 3 NULL 4.00 1.0
246-
d 3 2.0 3.00 0.0
239+
c 3 2.0 2 NULL
240+
a 1 1.0 1 NULL
241+
c 3 2.0 2 15.1
242+
b 2 1.0 1 10
243+
d 3 0.0 3 1
244+
NULL 3 0.0 3 1
245+
d 3 NULL 4 1
246+
d 3 2.0 3 0
247247

248248

249249
-- !query 16

‎sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -956,7 +956,7 @@ SELECT * FROM tt1
956956
struct<a:string,b:string,c:decimal(12,2),d:string>
957957
-- !query 83 output
958958
0123456789 abc 42.12 abc
959-
abc 0123456789 42.00 abcd
959+
abc 0123456789 42 abcd
960960

961961

962962
-- !query 84

0 commit comments

Comments
 (0)