Skip to content

[SPARK-36921][SQL] Support ANSI intervals by DIV #34257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-36921][SQL] Support ANSI intervals by DIV #34257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,8 @@ case class Divide(
Examples:
> SELECT 3 _FUNC_ 2;
1
> SELECT INTERVAL '1-1' YEAR TO MONTH _FUNC_ INTERVAL '-1' MONTH;
-13
""",
since = "3.0.0",
group = "math_funcs")
Expand All @@ -584,7 +586,8 @@ case class IntegralDivide(
case _ => false
}

override def inputType: AbstractDataType = TypeCollection(LongType, DecimalType)
override def inputType: AbstractDataType = TypeCollection(
LongType, DecimalType, YearMonthIntervalType, DayTimeIntervalType)

override def dataType: DataType = LongType

Expand All @@ -599,6 +602,10 @@ case class IntegralDivide(
i.integral.asInstanceOf[Integral[Any]]
case d: DecimalType =>
d.asIntegral.asInstanceOf[Integral[Any]]
case _: YearMonthIntervalType =>
IntegerType.integral.asInstanceOf[Integral[Any]]
case _: DayTimeIntervalType =>
LongType.integral.asInstanceOf[Integral[Any]]
}
(x, y) => {
val res = integral.quot(x, y)
Expand Down
81 changes: 81 additions & 0 deletions .../src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -699,4 +699,85 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkConsistencyBetweenInterpretedAndCodegen((e: Expression) => Abs(e, false), tpe)
}
}

test("SPARK-36921: Support YearMonthIntervalType by div") {
checkEvaluation(IntegralDivide(Literal(Period.ZERO), Literal(Period.ZERO)), null)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Literal(Period.ZERO)), null)
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
Literal(Period.ZERO)), null)
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
Literal(Period.ZERO)), null)

checkEvaluation(IntegralDivide(Literal.create(null, YearMonthIntervalType()),
Literal.create(null, YearMonthIntervalType())), null)
checkEvaluation(IntegralDivide(Literal.create(null, YearMonthIntervalType()),
Literal(Period.ofYears(1))), null)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Literal.create(null, YearMonthIntervalType())), null)

checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
Period.ofMonths(Int.MaxValue)), 1L)
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MaxValue),
Period.ofMonths(Int.MinValue)), 0L)
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
Period.ofMonths(Int.MinValue)), 1L)
checkEvaluation(IntegralDivide(Period.ofMonths(Int.MinValue),
Period.ofMonths(Int.MaxValue)), -1L)

checkEvaluation(IntegralDivide(Literal(Period.ZERO),
Literal(Period.ofYears(-1))), 0L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(2)),
Literal(Period.ofYears(1))), 2L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(2)),
Literal(Period.ofYears(-1))), -2L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Literal(Period.ofMonths(3))), 4L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Copy link
Member

@MaxGekk MaxGekk Oct 12, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you check negative intervals, and also some corner cases like division by max/min and zero intervals (ArithmeticExpressionSuite).

Literal(Period.ofMonths(-3))), -4L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Literal(Period.ofMonths(5))), 2L)
checkEvaluation(IntegralDivide(Literal(Period.ofYears(1)),
Literal(Period.ofMonths(-5))), -2L)
}
test("SPARK-36921: Support DayTimeIntervalType by div") {
checkEvaluation(IntegralDivide(Literal(Duration.ZERO), Literal(Duration.ZERO)), null)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal(Duration.ZERO)), null)
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
Literal(Duration.ZERO)), null)
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
Literal(Duration.ZERO)), null)

checkEvaluation(IntegralDivide(Literal.create(null, DayTimeIntervalType()),
Literal.create(null, DayTimeIntervalType())), null)
checkEvaluation(IntegralDivide(Literal.create(null, DayTimeIntervalType()),
Literal(Duration.ofDays(1))), null)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal.create(null, DayTimeIntervalType())), null)

checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS))), 1L)
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS))), 1L)
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS))), 0L)
checkEvaluation(IntegralDivide(Literal(Duration.of(Long.MinValue, ChronoUnit.MICROS)),
Literal(Duration.of(Long.MaxValue, ChronoUnit.MICROS))), -1L)

checkEvaluation(IntegralDivide(Literal(Duration.ZERO),
Literal(Duration.ofDays(-1))), 0L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(2)),
Literal(Duration.ofDays(1))), 2L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(2)),
Literal(Duration.ofDays(-1))), -2L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal(Duration.ofHours(4))), 6L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal(Duration.ofHours(-4))), -6L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal(Duration.ofHours(5))), 4L)
checkEvaluation(IntegralDivide(Literal(Duration.ofDays(1)),
Literal(Duration.ofHours(-5))), -4L)
}
}
5 changes: 5 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/interval.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,3 +371,8 @@ SELECT coalesce(INTERVAL '1' DAY, INTERVAL '01:01' HOUR TO MINUTE);
SELECT coalesce(INTERVAL 1 MONTH, INTERVAL 20 DAYS);
SELECT abs(INTERVAL '-10' YEAR);
SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND);
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR);
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH);
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY);
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR);
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY);
43 changes: 42 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 271
-- Number of queries: 276


-- !query
Expand Down Expand Up @@ -2568,3 +2568,44 @@ SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND)
struct<abs(INTERVAL '-1 02:03:04.123' DAY TO SECOND):interval day to second>
-- !query output
1 02:03:04.123000000


-- !query
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR)
-- !query schema
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '1' YEAR):bigint>
-- !query output
1


-- !query
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH)
-- !query schema
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '-1' MONTH):bigint>
-- !query output
-13


-- !query
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY)
-- !query schema
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '1' DAY):bigint>
-- !query output
1


-- !query
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR)
-- !query schema
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '-01' HOUR):bigint>
-- !query output
-30


-- !query
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY)
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
cannot resolve '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' due to data type mismatch: differing types in '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' (interval month and interval day).; line 1 pos 7
43 changes: 42 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/interval.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 271
-- Number of queries: 276


-- !query
Expand Down Expand Up @@ -2557,3 +2557,44 @@ SELECT abs(INTERVAL -'1 02:03:04.123' DAY TO SECOND)
struct<abs(INTERVAL '-1 02:03:04.123' DAY TO SECOND):interval day to second>
-- !query output
1 02:03:04.123000000


-- !query
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '1' YEAR)
-- !query schema
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '1' YEAR):bigint>
-- !query output
1


-- !query
SELECT div(INTERVAL '1-1' YEAR TO MONTH, INTERVAL '-1' MONTH)
-- !query schema
struct<(INTERVAL '1-1' YEAR TO MONTH div INTERVAL '-1' MONTH):bigint>
-- !query output
-13


-- !query
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '1' DAY)
-- !query schema
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '1' DAY):bigint>
-- !query output
1


-- !query
SELECT div(INTERVAL '1 06' DAY TO HOUR, INTERVAL '-1' HOUR)
-- !query schema
struct<(INTERVAL '1 06' DAY TO HOUR div INTERVAL '-01' HOUR):bigint>
-- !query output
-30


-- !query
SELECT div(INTERVAL '1' MONTH, INTERVAL '-1' DAY)
-- !query schema
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
cannot resolve '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' due to data type mismatch: differing types in '(INTERVAL '1' MONTH div INTERVAL '-1' DAY)' (interval month and interval day).; line 1 pos 7