Skip to content

Commit 52feb3f

Browse files
hvanhovellrxin
authored andcommitted
[SPARK-17246][SQL] Add BigDecimal literal
## What changes were proposed in this pull request? This PR adds parser support for `BigDecimal` literals. If you append the suffix `BD` to a valid number then this will be interpreted as a `BigDecimal`, for example `12.0E10BD` will interpreted into a BigDecimal with scale -9 and precision 3. This is useful in situations where you need exact values. ## How was this patch tested? Added tests to `ExpressionParserSuite`, `ExpressionSQLBuilderSuite` and `SQLQueryTestSuite`. Author: Herman van Hovell <[email protected]> Closes #14819 from hvanhovell/SPARK-17246. (cherry picked from commit a11d10f) Signed-off-by: Reynold Xin <[email protected]>
1 parent deb6a54 commit 52feb3f

File tree

7 files changed

+59
-3
lines changed

7 files changed

+59
-3
lines changed

sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,7 @@ number
626626
| MINUS? SMALLINT_LITERAL #smallIntLiteral
627627
| MINUS? TINYINT_LITERAL #tinyIntLiteral
628628
| MINUS? DOUBLE_LITERAL #doubleLiteral
629+
| MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral
629630
;
630631

631632
nonReserved
@@ -920,6 +921,11 @@ DOUBLE_LITERAL
920921
(INTEGER_VALUE | DECIMAL_VALUE | SCIENTIFIC_DECIMAL_VALUE) 'D'
921922
;
922923

924+
BIGDECIMAL_LITERAL
925+
:
926+
(INTEGER_VALUE | DECIMAL_VALUE | SCIENTIFIC_DECIMAL_VALUE) 'BD'
927+
;
928+
923929
IDENTIFIER
924930
: (LETTER | DIGIT | '_')+
925931
;

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression with
266266
case Double.NegativeInfinity => s"CAST('-Infinity' AS ${DoubleType.sql})"
267267
case _ => v + "D"
268268
}
269-
case (v: Decimal, t: DecimalType) => s"CAST($v AS ${t.sql})"
269+
case (v: Decimal, t: DecimalType) => v + "BD"
270270
case (v: Int, DateType) => s"DATE '${DateTimeUtils.toJavaDate(v)}'"
271271
case (v: Long, TimestampType) => s"TIMESTAMP('${DateTimeUtils.toJavaTimestamp(v)}')"
272272
case _ => value.toString

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ import org.antlr.v4.runtime.{ParserRuleContext, Token}
2626
import org.antlr.v4.runtime.tree.{ParseTree, RuleNode, TerminalNode}
2727

2828
import org.apache.spark.internal.Logging
29-
import org.apache.spark.sql.catalyst.{FunctionIdentifier, InternalRow, TableIdentifier}
29+
import org.apache.spark.sql.AnalysisException
30+
import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
3031
import org.apache.spark.sql.catalyst.analysis._
3132
import org.apache.spark.sql.catalyst.expressions._
3233
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
@@ -1318,6 +1319,19 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
13181319
numericLiteral(ctx, Double.MinValue, Double.MaxValue, DoubleType.simpleString)(_.toDouble)
13191320
}
13201321

1322+
/**
1323+
* Create a BigDecimal Literal expression.
1324+
*/
1325+
override def visitBigDecimalLiteral(ctx: BigDecimalLiteralContext): Literal = {
1326+
val raw = ctx.getText.substring(0, ctx.getText.length - 2)
1327+
try {
1328+
Literal(BigDecimal(raw).underlying())
1329+
} catch {
1330+
case e: AnalysisException =>
1331+
throw new ParseException(e.message, ctx)
1332+
}
1333+
}
1334+
13211335
/**
13221336
* Create a String literal expression.
13231337
*/

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,13 @@ class ExpressionParserSuite extends PlanTest {
392392
intercept("1.8E308D", s"does not fit in range")
393393
// TODO we need to figure out if we should throw an exception here!
394394
assertEqual("1E309", Literal(Double.PositiveInfinity))
395+
396+
// BigDecimal Literal
397+
assertEqual("90912830918230182310293801923652346786BD",
398+
Literal(BigDecimal("90912830918230182310293801923652346786").underlying()))
399+
assertEqual("123.0E-28BD", Literal(BigDecimal("123.0E-28").underlying()))
400+
assertEqual("123.08BD", Literal(BigDecimal("123.08").underlying()))
401+
intercept("1.20E-38BD", "DecimalType can only support precision up to 38")
395402
}
396403

397404
test("strings") {

sql/core/src/test/resources/sql-tests/inputs/literals.sql

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,9 @@ select interval 10 nanoseconds;
9090

9191
-- unsupported data type
9292
select GEO '(10,-6)';
93+
94+
-- big decimal parsing
95+
select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD;
96+
97+
-- out of range big decimal
98+
select 1.20E-38BD;

sql/core/src/test/resources/sql-tests/results/literals.sql.out

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 38
2+
-- Number of queries: 40
33

44

55
-- !query 0
@@ -354,3 +354,25 @@ Literals of type 'GEO' are currently not supported.(line 1, pos 7)
354354
== SQL ==
355355
select GEO '(10,-6)'
356356
-------^^^
357+
358+
359+
-- !query 38
360+
select 90912830918230182310293801923652346786BD, 123.0E-28BD, 123.08BD
361+
-- !query 38 schema
362+
struct<90912830918230182310293801923652346786:decimal(38,0),1.230E-26:decimal(29,29),123.08:decimal(5,2)>
363+
-- !query 38 output
364+
90912830918230182310293801923652346786 0.0000000000000000000000000123 123.08
365+
366+
367+
-- !query 39
368+
select 1.20E-38BD
369+
-- !query 39 schema
370+
struct<>
371+
-- !query 39 output
372+
org.apache.spark.sql.catalyst.parser.ParseException
373+
374+
DecimalType can only support precision up to 38(line 1, pos 7)
375+
376+
== SQL ==
377+
select 1.20E-38BD
378+
-------^^^

sql/hive/src/test/scala/org/apache/spark/sql/catalyst/ExpressionSQLBuilderSuite.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ class ExpressionSQLBuilderSuite extends SQLBuilderTest {
3939
checkSQL(Literal(Double.PositiveInfinity), "CAST('Infinity' AS DOUBLE)")
4040
checkSQL(Literal(Double.NegativeInfinity), "CAST('-Infinity' AS DOUBLE)")
4141
checkSQL(Literal(Double.NaN), "CAST('NaN' AS DOUBLE)")
42+
checkSQL(Literal(BigDecimal("10.0000000").underlying), "10.0000000BD")
4243
checkSQL(
4344
Literal(Timestamp.valueOf("2016-01-01 00:00:00")), "TIMESTAMP('2016-01-01 00:00:00.0')")
4445
// TODO tests for decimals

0 commit comments

Comments
 (0)