Skip to content

Commit 1ccc4dc

Browse files
Peng-LeiMaxGekk
Peng-Lei
authored and
MaxGekk
committed
[SPARK-36921][SQL] Support ANSI intervals by DIV
### What changes were proposed in this pull request? 1. support div(YearMonthIntervalType, YearMonthIntervalType), return long result 2. support div(DayTimeIntervalType, DayTimeIntervalType), return long result 3. if input is NULL or input2 is 0, then return null ### Why are the changes needed? Extended the div function to support ANSI intervals. The operation should produce quotient of division. [SPARK-36921](https://issues.apache.org/jira/browse/SPARK-36921) ### Does this PR introduce _any_ user-facing change? Yes, user can use user can use YearMonthIntervalType and DayTimeIntervalType as input for div function. ### How was this patch tested? Add ut testcase Closes #34257 from Peng-Lei/SPARK-36921. Authored-by: PengLei <[email protected]> Signed-off-by: Max Gekk <[email protected]>
1 parent 9290924 commit 1ccc4dc

File tree

5 files changed

+178
-3
lines changed

5 files changed

+178
-3
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,8 @@ case class Divide(
568568
Examples:
569569
> SELECT 3 _FUNC_ 2;
570570
1
571+
> SELECT INTERVAL '1-1' YEAR TO MONTH _FUNC_ INTERVAL '-1' MONTH;
572+
-13
571573
""",
572574
since = "3.0.0",
573575
group = "math_funcs")
@@ -584,7 +586,8 @@ case class IntegralDivide(
584586
case _ => false
585587
}
586588

587-
override def inputType: AbstractDataType = TypeCollection(LongType, DecimalType)
589+
override def inputType: AbstractDataType = TypeCollection(
590+
LongType, DecimalType, YearMonthIntervalType, DayTimeIntervalType)
588591

589592
override def dataType: DataType = LongType
590593

@@ -599,6 +602,10 @@ case class IntegralDivide(
599602
i.integral.asInstanceOf[Integral[Any]]
600603
case d: DecimalType =>
601604
d.asIntegral.asInstanceOf[Integral[Any]]
605+
case _: YearMonthIntervalType =>
606+
IntegerType.integral.asInstanceOf[Integral[Any]]
607+
case _: DayTimeIntervalType =>
608+
LongType.integral.asInstanceOf[Integral[Any]]
602609
}
603610
(x, y) => {
604611
val res = integral.quot(x, y)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -699,4 +699,85 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
699699
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Abs(e, false), tpe)
700700
}
701701
}
702+
703+
test("SPARK-36921: Support YearMonthIntervalType by div") {
704+
checkEvaluation(IntegralDivide(Literal(Period.ZERO), Literal(Period.ZERO)), null)
705+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
706+
Literal(Period.ZERO)), null)
707+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
708+
Literal(Period.ZERO)), null)
709+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
710+
Literal(Period.ZERO)), null)
711+
712+
checkEvaluation(IntegralDivide(Literal.create(null, YearMonthIntervalType()),
713+
Literal.create(null, YearMonthIntervalType())), null)
714+
checkEvaluation(IntegralDivide(Literal.create(null, YearMonthIntervalType()),
715+
Literal(Period.ofYears(1))), null)
716+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
717+
Literal.create(null, YearMonthIntervalType())), null)
718+
719+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
720+
Period.ofMonths(Int.MaxValue)), 1L)
721+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
722+
Period.ofMonths(Int.MinValue)), 0L)
723+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
724+
Period.ofMonths(Int.MinValue)), 1L)
725+
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
726+
Period.ofMonths(Int.MaxValue)), -1L)
727+
728+
checkEvaluation(IntegralDivide(Literal(Period.ZERO),
729+
Literal(Period.ofYears(-1))), 0L)
730+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(2)),
731+
Literal(Period.ofYears(1))), 2L)
732+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(2)),
733+
Literal(Period.ofYears(-1))), -2L)
734+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
735+
Literal(Period.ofMonths(3))), 4L)
736+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
737+
Literal(Period.ofMonths(-3))), -4L)
738+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
739+
Literal(Period.ofMonths(5))), 2L)
740+
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
741+
Literal(Period.ofMonths(-5))), -2L)
742+
}
743+
test("SPARK-36921: Support DayTimeIntervalType by div") {
744+
checkEvaluation(IntegralDivide(Literal(Duration.ZERO), Literal(Duration.ZERO)), null)
745+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
746+
Literal(Duration.ZERO)), null)
747+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
748+
Literal(Duration.ZERO)), null)
749+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
750+
Literal(Duration.ZERO)), null)
751+
752+
checkEvaluation(IntegralDivide(Literal.create(null, DayTimeIntervalType()),
753+
Literal.create(null, DayTimeIntervalType())), null)
754+
checkEvaluation(IntegralDivide(Literal.create(null, DayTimeIntervalType()),
755+
Literal(Duration.ofDays(1))), null)
756+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
757+
Literal.create(null, DayTimeIntervalType())), null)
758+
759+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
760+
Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS))), 1L)
761+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
762+
Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS))), 1L)
763+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
764+
Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS))), 0L)
765+
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
766+
Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS))), -1L)
767+
768+
checkEvaluation(IntegralDivide(Literal(Duration.ZERO),
769+
Literal(Duration.ofDays(-1))), 0L)
770+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(2)),
771+
Literal(Duration.ofDays(1))), 2L)
772+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(2)),
773+
Literal(Duration.ofDays(-1))), -2L)
774+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
775+
Literal(Duration.ofHours(4))), 6L)
776+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
777+
Literal(Duration.ofHours(-4))), -6L)
778+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
779+
Literal(Duration.ofHours(5))), 4L)
780+
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
781+
Literal(Duration.ofHours(-5))), -4L)
782+
}
702783
}

‎sql/core/src/test/resources/sql-tests/inputs/interval.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,3 +371,8 @@ SELECT coalesce(INTERVAL '1' DAY, INTERVAL '01:01' HOUR TO MINUTE);
371371
SELECT coalesce(INTERVAL 1 MONTH, INTERVAL 20 DAYS);
372372
SELECT abs(INTERVAL '-10' YEAR);
373373
SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND);
374+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR);
375+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH);
376+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY);
377+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR);
378+
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY);

‎sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 271
2+
-- Number of queries: 276
33

44

55
-- !query
@@ -2568,3 +2568,44 @@ SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND)
25682568
struct<abs(INTERVAL '-1 02:03:04.123' DAY TO SECOND):interval day to second>
25692569
-- !query output
25702570
1 02:03:04.123000000
2571+
2572+
2573+
-- !query
2574+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR)
2575+
-- !query schema
2576+
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '1' YEAR):bigint>
2577+
-- !query output
2578+
1
2579+
2580+
2581+
-- !query
2582+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH)
2583+
-- !query schema
2584+
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '-1' MONTH):bigint>
2585+
-- !query output
2586+
-13
2587+
2588+
2589+
-- !query
2590+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY)
2591+
-- !query schema
2592+
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '1' DAY):bigint>
2593+
-- !query output
2594+
1
2595+
2596+
2597+
-- !query
2598+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR)
2599+
-- !query schema
2600+
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '-01' HOUR):bigint>
2601+
-- !query output
2602+
-30
2603+
2604+
2605+
-- !query
2606+
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY)
2607+
-- !query schema
2608+
struct<>
2609+
-- !query output
2610+
org.apache.spark.sql.AnalysisException
2611+
cannot resolve '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' due to data type mismatch: differing types in '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' (interval month and interval day).; line 1 pos 7

‎sql/core/src/test/resources/sql-tests/results/interval.sql.out‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 271
2+
-- Number of queries: 276
33

44

55
-- !query
@@ -2557,3 +2557,44 @@ SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND)
25572557
struct<abs(INTERVAL '-1 02:03:04.123' DAY TO SECOND):interval day to second>
25582558
-- !query output
25592559
1 02:03:04.123000000
2560+
2561+
2562+
-- !query
2563+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR)
2564+
-- !query schema
2565+
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '1' YEAR):bigint>
2566+
-- !query output
2567+
1
2568+
2569+
2570+
-- !query
2571+
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH)
2572+
-- !query schema
2573+
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '-1' MONTH):bigint>
2574+
-- !query output
2575+
-13
2576+
2577+
2578+
-- !query
2579+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY)
2580+
-- !query schema
2581+
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '1' DAY):bigint>
2582+
-- !query output
2583+
1
2584+
2585+
2586+
-- !query
2587+
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR)
2588+
-- !query schema
2589+
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '-01' HOUR):bigint>
2590+
-- !query output
2591+
-30
2592+
2593+
2594+
-- !query
2595+
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY)
2596+
-- !query schema
2597+
struct<>
2598+
-- !query output
2599+
org.apache.spark.sql.AnalysisException
2600+
cannot resolve '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' due to data type mismatch: differing types in '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' (interval month and interval day).; line 1 pos 7

0 commit comments

Comments
 (0)