with q/Q

Author: yaooqinn

docs/sql-ref-datetime-pattern.md

@@ -137,7 +137,7 @@ The count of pattern letters determines the format.
   During parsing, the whole section may be missing from the parsed string.
   An optional section is started by `[` and ended using `]` (or at the end of the pattern).
   
-- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F' can only be used for datetime formatting, e.g. `date_format`.
+- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`.
 
 More details for the text style:
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala

@@ -228,10 +228,11 @@ private object DateTimeFormatterHelper {
   }
   final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p')
   // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values
-  // to timestamp, especially when they are mixed with other non-week-based ones. we have tried our
-  // best to restore the behavior change between 2.4 and 3.0 and failed, see
-  // https://github.com/apache/spark/pull/28674
-  final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F')
+  // to datetime, especially when they are mixed with other non-week-based ones
+  // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
+  // be directly resolved then the `q` do check for whether the month is valid, but if the date
+  // fields is incomplete, e.g. `yM`, the checking will be bypassed.
+  final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q')
   final val unsupportedPatternLengths = {
     // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in
     // 2.4

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala

@@ -1169,7 +1169,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       MillisToTimestamp(Literal(-92233720368547758L)), "long overflow")
   }
 
-  test("Disable week-based date filed for parsing") {
+  test("Disable week-based date fields and quarter fields for parsing") {
 
     def checkSparkUpgrade(c: Char): Unit = {
       checkExceptionInExpression[SparkUpgradeException](
@@ -1182,7 +1182,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
         UnixTimestamp(Literal("1"), Literal(c.toString)), "3.0")
     }
 
-    Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l =>
+    Seq('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q').foreach { l =>
       checkSparkUpgrade(l)
     }
   }