support string literal as the second argument of date_add/date_sub functions

Author: cloud-fan

docs/sql-migration-guide.md

@@ -113,7 +113,7 @@ license: |
 
 ### UDFs and Built-in Functions
 
-  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
+  - Since Spark 3.0, the `date_add` and `date_sub` functions only accepts int, smallint, tinyint as the 2nd argument, fractional and string types (excluding string literals) are not valid anymore, e.g. `date_add(cast('1964-05-23' as date), '12.34')` will cause `AnalysisException`. In Spark version 2.4 and earlier, if the 2nd argument is fractional or string value, it will be coerced to int value, and the result will be a date value of `1964-06-04`.
 
   - Since Spark 3.0, the function `percentile_approx` and its alias `approx_percentile` only accept integral value with range in `[1, 2147483647]` as its 3rd argument `accuracy`, fractional and string types are disallowed, e.g. `percentile_approx(10.0, 0.2, 1.8D)` will cause `AnalysisException`. In Spark version 2.4 and earlier, if `accuracy` is fractional or string value, it will be coerced to an int value, `percentile_approx(10.0, 0.2, 1.8D)` is operated as `percentile_approx(10.0, 0.2, 1)` which results in `10.0`.
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala

@@ -63,6 +63,7 @@ object TypeCoercion {
       ImplicitTypeCasts ::
       DateTimeOperations ::
       WindowFrameCoercion ::
+      StringLiteralCoercion ::
       Nil
 
   // See https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types.
@@ -1043,6 +1044,22 @@ object TypeCoercion {
       }
     }
   }
+
+  /**
+   * A special rule to support string literal as the second argument of date_add/date_sub functions,
+   * to keep backward compatibility.
+   * TODO: revisit the type coercion rules for string.
+   */
+  object StringLiteralCoercion extends TypeCoercionRule {
+    override protected def coerceTypes(plan: LogicalPlan): LogicalPlan = plan resolveExpressions {
+      // Skip nodes who's children have not been resolved yet.
+      case e if !e.childrenResolved => e
+      case DateAdd(l, r) if r.dataType == StringType && r.foldable =>
+        DateAdd(l, AnsiCast(r, IntegerType))
+      case DateSub(l, r) if r.dataType == StringType && r.foldable =>
+        DateSub(l, AnsiCast(r, IntegerType))
+    }
+  }
 }
 
 trait TypeCoercionRule extends Rule[LogicalPlan] with Logging {

sql/core/src/test/resources/sql-tests/inputs/datetime.sql

@@ -58,9 +58,12 @@ select date_add('2011-11-11', 1L);
 select date_add('2011-11-11', 1.0);
 select date_add('2011-11-11', 1E1);
 select date_add('2011-11-11', '1');
+select date_add('2011-11-11', '1.2');
 select date_add(date'2011-11-11', 1);
 select date_add(timestamp'2011-11-11', 1);
 select date_sub(date'2011-11-11', 1);
+select date_sub(date'2011-11-11', '1');
+select date_sub(date'2011-11-11', '1.2');
 select date_sub(timestamp'2011-11-11', 1);
 select date_sub(null, 1);
 select date_sub(date'2011-11-11', null);
@@ -72,6 +75,11 @@ select date '2001-10-01' - 7;
 select date '2001-09-28' + null;
 select date '2001-09-28' - null;
 
+-- date add/sub with non-literal string column
+create temp view v as select '1' str;
+select date_add('2011-11-11', str) from v;
+select date_sub('2011-11-11', str) from v;
+
 -- subtract dates
 select null - date '2019-10-06';
 select date '2001-10-01' - date '2001-09-28';

sql/core/src/test/resources/sql-tests/results/datetime.sql.out

@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 77
+-- Number of queries: 83
 
 
 -- !query
@@ -266,10 +266,18 @@ cannot resolve 'date_add(CAST('2011-11-11' AS DATE), 10.0D)' due to data type mi
 -- !query
 select date_add('2011-11-11', '1')
 -- !query schema
+struct<date_add(CAST(2011-11-11 AS DATE), CAST(1 AS INT)):date>
+-- !query output
+2011-11-12
+
+
+-- !query
+select date_add('2011-11-11', '1.2')
+-- !query schema
 struct<>
 -- !query output
-org.apache.spark.sql.AnalysisException
-cannot resolve 'date_add(CAST('2011-11-11' AS DATE), '1')' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, ''1'' is of string type.; line 1 pos 7
+java.lang.NumberFormatException
+invalid input syntax for type numeric: 1.2
 
 
 -- !query
@@ -296,6 +304,23 @@ struct<date_sub(DATE '2011-11-11', 1):date>
 2011-11-10
 
 
+-- !query
+select date_sub(date'2011-11-11', '1')
+-- !query schema
+struct<date_sub(DATE '2011-11-11', CAST(1 AS INT)):date>
+-- !query output
+2011-11-10
+
+
+-- !query
+select date_sub(date'2011-11-11', '1.2')
+-- !query schema
+struct<>
+-- !query output
+java.lang.NumberFormatException
+invalid input syntax for type numeric: 1.2
+
+
 -- !query
 select date_sub(timestamp'2011-11-11', 1)
 -- !query schema
@@ -377,6 +402,32 @@ struct<date_sub(DATE '2001-09-28', CAST(NULL AS INT)):date>
 NULL
 
 
+-- !query
+create temp view v as select '1' str
+-- !query schema
+struct<>
+-- !query output
+
+
+
+-- !query
+select date_add('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_add(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
+-- !query
+select date_sub('2011-11-11', str) from v
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.AnalysisException
+cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.`str`)' due to data type mismatch: argument 2 requires (int or smallint or tinyint) type, however, 'v.`str`' is of string type.; line 1 pos 7
+
+
 -- !query
 select null - date '2019-10-06'
 -- !query schema