Add config spark.sql.legacy.datesSubtraction.enabled

Author: MaxGekk

docs/sql-migration-guide.md

@@ -217,7 +217,7 @@ license: |
 
   - Since Spark 3.0, the `size` function returns `NULL` for the `NULL` input. In Spark version 2.4 and earlier, this function gives `-1` for the same input. To restore the behavior before Spark 3.0, you can set `spark.sql.legacy.sizeOfNull` to `true`.
 
-  - In Spark version 2.4 and earlier, dates subtraction `date1` - `date2` gives the number of days from `date1` to `date2`. Since Spark 3.0, the expression has the `INTERVAL` type and returns an interval between two dates. To get the number of days, use the `datediff` function.
+  - In Spark version 2.4 and earlier, dates subtraction `date1` - `date2` gives the number of days from `date1` to `date2`. Since Spark 3.0, the expression has the `INTERVAL` type and returns an interval between two dates. To get the number of days, you can set `spark.sql.legacy.datesSubtraction.enabled` to `true`.
 
 ## Upgrading from Spark SQL 2.4 to 2.4.1
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala

@@ -849,7 +849,12 @@ object TypeCoercion {
       case Add(l @ DateType(), r @ IntegerType()) => DateAdd(l, r)
       case Add(l @ IntegerType(), r @ DateType()) => DateAdd(r, l)
       case Subtract(l @ DateType(), r @ IntegerType()) => DateSub(l, r)
-      case Subtract(l @ DateType(), r @ DateType()) => SubtractDates(l, r)
+      case Subtract(l @ DateType(), r @ DateType()) =>
+        if (SQLConf.get.getConf(SQLConf.LEGACY_DATES_SUBTRACTION)) {
+          DateDiff(l, r)
+        } else {
+          SubtractDates(l, r)
+        }
       case Subtract(l @ TimestampType(), r @ TimestampType()) => TimestampDiff(l, r)
       case Subtract(l @ TimestampType(), r @ DateType()) =>
         TimestampDiff(l, Cast(r, TimestampType))

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

@@ -2014,6 +2014,15 @@ object SQLConf {
       .stringConf
       .createWithDefault(
         "https://maven-central.storage-download.googleapis.com/repos/central/data/")
+
+  val LEGACY_DATES_SUBTRACTION =
+    buildConf("spark.sql.legacy.datesSubtraction.enabled")
+      .doc("When true, date subtraction expressions have the INT type and return " +
+        "the number of days between the dates. If it is set to false, the expressions " +
+        "have the INTERVAL type and return an interval from the left date (inclusive) " +
+        "to the right date (exclusive).")
+      .booleanConf
+      .createWithDefault(false)
 }
 
 /**

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala

@@ -1430,7 +1430,12 @@ class TypeCoercionSuite extends AnalysisTest {
     ruleTest(dateTimeOperations, Add(date, intValue), DateAdd(date, intValue))
     ruleTest(dateTimeOperations, Add(intValue, date), DateAdd(date, intValue))
     ruleTest(dateTimeOperations, Subtract(date, intValue), DateSub(date, intValue))
-    ruleTest(dateTimeOperations, Subtract(date, date), SubtractDates(date, date))
+    withSQLConf(SQLConf.LEGACY_DATES_SUBTRACTION.key -> "false") {
+      ruleTest(dateTimeOperations, Subtract(date, date), SubtractDates(date, date))
+    }
+    withSQLConf(SQLConf.LEGACY_DATES_SUBTRACTION.key -> "true") {
+      ruleTest(dateTimeOperations, Subtract(date, date), DateDiff(date, date))
+    }
     ruleTest(dateTimeOperations, Subtract(timestamp, timestamp),
       TimestampDiff(timestamp, timestamp))
     ruleTest(dateTimeOperations, Subtract(timestamp, date),