Skip to content

Commit 2f72ebb

Browse files
mgaido91mgaido91
committed
[SPARK-24468][SQL] Handle negative scale when adjusting precision for decimal operations
1 parent 2c2a86b commit 2f72ebb

File tree

4 files changed

+96
-65
lines changed

4 files changed

+96
-65
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,13 +161,17 @@ object DecimalType extends AbstractDataType {
161161
* This method is used only when `spark.sql.decimalOperations.allowPrecisionLoss` is set to true.
162162
*/
163163
private[sql] def adjustPrecisionScale(precision: Int, scale: Int): DecimalType = {
164-
// Assumptions:
164+
// Assumption:
165165
assert(precision >= scale)
166-
assert(scale >= 0)
167166

168167
if (precision <= MAX_PRECISION) {
169168
// Adjustment only needed when we exceed max precision
170169
DecimalType(precision, scale)
170+
} else if (scale < 0) {
171+
// Decimal can have negative scale (SPARK-24468). In this case, we cannot allow a precision
172+
// loss since we would cause a loss of digits in the integer part.
173+
// In this case, we are likely to meet an overflow.
174+
DecimalType(MAX_PRECISION, scale)
171175
} else {
172176
// Precision/scale exceed maximum precision. Result must be adjusted to MAX_PRECISION.
173177
val intDigits = precision - scale

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionSuite.scala‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,15 @@ class DecimalPrecisionSuite extends AnalysisTest with BeforeAndAfter {
272272
}
273273
}
274274

275+
test("SPARK-24468: operations on decimals with negative scale") {
276+
val a = AttributeReference("a", DecimalType(3, -10))()
277+
val b = AttributeReference("b", DecimalType(1, -1))()
278+
val c = AttributeReference("c", DecimalType(35, 1))()
279+
checkType(Multiply(a, b), DecimalType(5, -11))
280+
checkType(Multiply(a, c), DecimalType(38, -9))
281+
checkType(Multiply(b, c), DecimalType(37, 0))
282+
}
283+
275284
/** strength reduction for integer/decimal comparisons */
276285
def ruleTest(initial: Expression, transformed: Expression): Unit = {
277286
val testRelation = LocalRelation(AttributeReference("a", IntegerType)())

‎sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/decimalArithmeticOperations.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ select 10.3000 * 3.0;
4040
select 10.30000 * 30.0;
4141
select 10.300000000000000000 * 3.000000000000000000;
4242
select 10.300000000000000000 * 3.0000000000000000000;
43+
select 2.35E10 * 1.0;
4344

4445
-- arithmetic operations causing an overflow return NULL
4546
select (5e36 + 0.1) + 5e36;
@@ -67,6 +68,7 @@ select 10.3000 * 3.0;
6768
select 10.30000 * 30.0;
6869
select 10.300000000000000000 * 3.000000000000000000;
6970
select 10.300000000000000000 * 3.0000000000000000000;
71+
select 2.35E10 * 1.0;
7072

7173
-- arithmetic operations causing an overflow return NULL
7274
select (5e36 + 0.1) + 5e36;

‎sql/core/src/test/resources/sql-tests/results/typeCoercion/native/decimalArithmeticOperations.sql.out‎

Lines changed: 79 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 36
2+
-- Number of queries: 38
33

44

55
-- !query 0
@@ -114,190 +114,206 @@ struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.00000000000000000
114114

115115

116116
-- !query 13
117-
select (5e36 + 0.1) + 5e36
117+
select 2.35E10 * 1.0
118118
-- !query 13 schema
119-
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
119+
struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)>
120120
-- !query 13 output
121-
NULL
121+
23500000000
122122

123123

124124
-- !query 14
125-
select (-4e36 - 0.1) - 7e36
125+
select (5e36 + 0.1) + 5e36
126126
-- !query 14 schema
127-
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
127+
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
128128
-- !query 14 output
129129
NULL
130130

131131

132132
-- !query 15
133-
select 12345678901234567890.0 * 12345678901234567890.0
133+
select (-4e36 - 0.1) - 7e36
134134
-- !query 15 schema
135-
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
135+
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
136136
-- !query 15 output
137137
NULL
138138

139139

140140
-- !query 16
141-
select 1e35 / 0.1
141+
select 12345678901234567890.0 * 12345678901234567890.0
142142
-- !query 16 schema
143-
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,6)>
143+
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
144144
-- !query 16 output
145145
NULL
146146

147147

148148
-- !query 17
149-
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
149+
select 1e35 / 0.1
150150
-- !query 17 schema
151-
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,6)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,6))):decimal(38,6)>
151+
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,6)>
152152
-- !query 17 output
153-
10012345678912345678912345678911.246907
153+
NULL
154154

155155

156156
-- !query 18
157-
select 123456789123456789.1234567890 * 1.123456789123456789
157+
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
158158
-- !query 18 schema
159-
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
159+
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,6)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,6))):decimal(38,6)>
160160
-- !query 18 output
161-
138698367904130467.654320988515622621
161+
10012345678912345678912345678911.246907
162162

163163

164164
-- !query 19
165-
select 12345678912345.123456789123 / 0.000000012345678
165+
select 123456789123456789.1234567890 * 1.123456789123456789
166166
-- !query 19 schema
167-
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,9)>
167+
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,18)>
168168
-- !query 19 output
169-
1000000073899961059796.725866332
169+
138698367904130467.654320988515622621
170170

171171

172172
-- !query 20
173-
set spark.sql.decimalOperations.allowPrecisionLoss=false
173+
select 12345678912345.123456789123 / 0.000000012345678
174174
-- !query 20 schema
175-
struct<key:string,value:string>
175+
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,9)>
176176
-- !query 20 output
177-
spark.sql.decimalOperations.allowPrecisionLoss false
177+
1000000073899961059796.725866332
178178

179179

180180
-- !query 21
181-
select id, a+b, a-b, a*b, a/b from decimals_test order by id
181+
set spark.sql.decimalOperations.allowPrecisionLoss=false
182182
-- !query 21 schema
183-
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
183+
struct<key:string,value:string>
184184
-- !query 21 output
185+
spark.sql.decimalOperations.allowPrecisionLoss false
186+
187+
188+
-- !query 22
189+
select id, a+b, a-b, a*b, a/b from decimals_test order by id
190+
-- !query 22 schema
191+
struct<id:int,(a + b):decimal(38,18),(a - b):decimal(38,18),(a * b):decimal(38,36),(a / b):decimal(38,18)>
192+
-- !query 22 output
185193
1 1099 -899 NULL 0.1001001001001001
186194
2 24690.246 0 NULL 1
187195
3 1234.2234567891011 -1233.9765432108989 NULL 0.000100037913541123
188196
4 123456789123456790.123456789123456789 123456789123456787.876543210876543211 NULL 109890109097814272.043109406191131436
189197

190198

191-
-- !query 22
199+
-- !query 23
192200
select id, a*10, b/10 from decimals_test order by id
193-
-- !query 22 schema
201+
-- !query 23 schema
194202
struct<id:int,(CAST(a AS DECIMAL(38,18)) * CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,18),(CAST(b AS DECIMAL(38,18)) / CAST(CAST(10 AS DECIMAL(2,0)) AS DECIMAL(38,18))):decimal(38,19)>
195-
-- !query 22 output
203+
-- !query 23 output
196204
1 1000 99.9
197205
2 123451.23 1234.5123
198206
3 1.234567891011 123.41
199207
4 1234567891234567890 0.1123456789123456789
200208

201209

202-
-- !query 23
203-
select 10.3 * 3.0
204-
-- !query 23 schema
205-
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
206-
-- !query 23 output
207-
30.9
208-
209-
210210
-- !query 24
211-
select 10.3000 * 3.0
211+
select 10.3 * 3.0
212212
-- !query 24 schema
213-
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
213+
struct<(CAST(10.3 AS DECIMAL(3,1)) * CAST(3.0 AS DECIMAL(3,1))):decimal(6,2)>
214214
-- !query 24 output
215215
30.9
216216

217217

218218
-- !query 25
219-
select 10.30000 * 30.0
219+
select 10.3000 * 3.0
220220
-- !query 25 schema
221-
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
221+
struct<(CAST(10.3000 AS DECIMAL(6,4)) * CAST(3.0 AS DECIMAL(6,4))):decimal(9,5)>
222222
-- !query 25 output
223-
309
223+
30.9
224224

225225

226226
-- !query 26
227-
select 10.300000000000000000 * 3.000000000000000000
227+
select 10.30000 * 30.0
228228
-- !query 26 schema
229-
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
229+
struct<(CAST(10.30000 AS DECIMAL(7,5)) * CAST(30.0 AS DECIMAL(7,5))):decimal(11,6)>
230230
-- !query 26 output
231-
30.9
231+
309
232232

233233

234234
-- !query 27
235-
select 10.300000000000000000 * 3.0000000000000000000
235+
select 10.300000000000000000 * 3.000000000000000000
236236
-- !query 27 schema
237-
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
237+
struct<(CAST(10.300000000000000000 AS DECIMAL(20,18)) * CAST(3.000000000000000000 AS DECIMAL(20,18))):decimal(38,36)>
238238
-- !query 27 output
239-
NULL
239+
30.9
240240

241241

242242
-- !query 28
243-
select (5e36 + 0.1) + 5e36
243+
select 10.300000000000000000 * 3.0000000000000000000
244244
-- !query 28 schema
245-
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
245+
struct<(CAST(10.300000000000000000 AS DECIMAL(21,19)) * CAST(3.0000000000000000000 AS DECIMAL(21,19))):decimal(38,37)>
246246
-- !query 28 output
247247
NULL
248248

249249

250250
-- !query 29
251-
select (-4e36 - 0.1) - 7e36
251+
select 2.35E10 * 1.0
252252
-- !query 29 schema
253-
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
253+
struct<(CAST(2.35E+10 AS DECIMAL(12,1)) * CAST(1.0 AS DECIMAL(12,1))):decimal(6,-7)>
254254
-- !query 29 output
255-
NULL
255+
23500000000
256256

257257

258258
-- !query 30
259-
select 12345678901234567890.0 * 12345678901234567890.0
259+
select (5e36 + 0.1) + 5e36
260260
-- !query 30 schema
261-
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
261+
struct<(CAST((CAST(5E+36 AS DECIMAL(38,1)) + CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) + CAST(5E+36 AS DECIMAL(38,1))):decimal(38,1)>
262262
-- !query 30 output
263263
NULL
264264

265265

266266
-- !query 31
267-
select 1e35 / 0.1
267+
select (-4e36 - 0.1) - 7e36
268268
-- !query 31 schema
269-
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
269+
struct<(CAST((CAST(-4E+36 AS DECIMAL(38,1)) - CAST(0.1 AS DECIMAL(38,1))) AS DECIMAL(38,1)) - CAST(7E+36 AS DECIMAL(38,1))):decimal(38,1)>
270270
-- !query 31 output
271271
NULL
272272

273273

274274
-- !query 32
275-
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
275+
select 12345678901234567890.0 * 12345678901234567890.0
276276
-- !query 32 schema
277-
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)>
277+
struct<(12345678901234567890.0 * 12345678901234567890.0):decimal(38,2)>
278278
-- !query 32 output
279279
NULL
280280

281281

282282
-- !query 33
283-
select 123456789123456789.1234567890 * 1.123456789123456789
283+
select 1e35 / 0.1
284284
-- !query 33 schema
285-
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
285+
struct<(CAST(1E+35 AS DECIMAL(37,1)) / CAST(0.1 AS DECIMAL(37,1))):decimal(38,3)>
286286
-- !query 33 output
287287
NULL
288288

289289

290290
-- !query 34
291-
select 12345678912345.123456789123 / 0.000000012345678
291+
select 12345678912345678912345678912.1234567 + 9999999999999999999999999999999.12345
292292
-- !query 34 schema
293-
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)>
293+
struct<(CAST(12345678912345678912345678912.1234567 AS DECIMAL(38,7)) + CAST(9999999999999999999999999999999.12345 AS DECIMAL(38,7))):decimal(38,7)>
294294
-- !query 34 output
295295
NULL
296296

297297

298298
-- !query 35
299-
drop table decimals_test
299+
select 123456789123456789.1234567890 * 1.123456789123456789
300300
-- !query 35 schema
301-
struct<>
301+
struct<(CAST(123456789123456789.1234567890 AS DECIMAL(36,18)) * CAST(1.123456789123456789 AS DECIMAL(36,18))):decimal(38,28)>
302302
-- !query 35 output
303+
NULL
304+
305+
306+
-- !query 36
307+
select 12345678912345.123456789123 / 0.000000012345678
308+
-- !query 36 schema
309+
struct<(CAST(12345678912345.123456789123 AS DECIMAL(29,15)) / CAST(1.2345678E-8 AS DECIMAL(29,15))):decimal(38,18)>
310+
-- !query 36 output
311+
NULL
312+
313+
314+
-- !query 37
315+
drop table decimals_test
316+
-- !query 37 schema
317+
struct<>
318+
-- !query 37 output
303319

0 commit comments

Comments
 (0)