diff --git a/docs/sql-ref-datetime-pattern.md b/docs/sql-ref-datetime-pattern.md index 865b9470ed191..5859ad82525f2 100644 --- a/docs/sql-ref-datetime-pattern.md +++ b/docs/sql-ref-datetime-pattern.md @@ -136,6 +136,8 @@ The count of pattern letters determines the format. During formatting, all valid data will be output even it is in the optional section. During parsing, the whole section may be missing from the parsed string. An optional section is started by `[` and ended using `]` (or at the end of the pattern). + +- Symbols of 'Y', 'W', 'w', 'E', 'u', 'F', 'q' and 'Q' can only be used for datetime formatting, e.g. `date_format`. They are not allowed used for datetime parsing, e.g. `to_timestamp`. More details for the text style: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index d02776b5d86f8..4e63ee7428d72 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -525,7 +525,7 @@ object CatalogColumnStat extends Logging { TimestampFormatter( format = "yyyy-MM-dd HH:mm:ss.SSSSSS", zoneId = ZoneOffset.UTC, - needVarLengthSecondFraction = isParsing) + isParsing = isParsing) } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala index a7c243537acb7..f0df18da8eed6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala @@ -35,7 +35,7 @@ class CSVInferSchema(val options: CSVOptions) extends Serializable { options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) private val decimalParser = if (options.locale == Locale.US) { // Special handling the default locale for backward compatibility diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala index 4990da2bf3797..a3ee129cd6d64 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala @@ -47,12 +47,13 @@ class UnivocityGenerator( options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) private val dateFormatter = DateFormatter( options.dateFormat, options.zoneId, options.locale, - legacyFormat = FAST_DATE_FORMAT) + legacyFormat = FAST_DATE_FORMAT, + isParsing = false) private def makeConverter(dataType: DataType): ValueConverter = dataType match { case DateType => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala index f2bb7db895ca2..3898eca79478e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityParser.scala @@ -90,12 +90,13 @@ class UnivocityParser( options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) private lazy val dateFormatter = DateFormatter( options.dateFormat, options.zoneId, options.locale, - legacyFormat = FAST_DATE_FORMAT) + legacyFormat = FAST_DATE_FORMAT, + isParsing = true) private val csvFilters = new CSVFilters(filters, requiredSchema) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 4f3db1b8a57ce..c5ead9412a438 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -734,7 +734,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) } } else None } @@ -745,7 +745,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti format.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) } else { formatter.get } @@ -890,7 +890,7 @@ abstract class ToTimestamp constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) } catch { case e: SparkUpgradeException => throw e case NonFatal(_) => null @@ -929,7 +929,7 @@ abstract class ToTimestamp formatString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) .parse(t.asInstanceOf[UTF8String].toString) / downScaleFactor } catch { case e: SparkUpgradeException => throw e @@ -1072,7 +1072,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ constFormat.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) } catch { case e: SparkUpgradeException => throw e case NonFatal(_) => null @@ -1105,7 +1105,7 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[ f.toString, zoneId, legacyFormat = SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) .format(time.asInstanceOf[Long] * MICROS_PER_SECOND)) } catch { case e: SparkUpgradeException => throw e diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala index f3938feef0a35..fb0ca323af1ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonGenerator.scala @@ -83,12 +83,13 @@ private[sql] class JacksonGenerator( options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = false) + isParsing = false) private val dateFormatter = DateFormatter( options.dateFormat, options.zoneId, options.locale, - legacyFormat = FAST_DATE_FORMAT) + legacyFormat = FAST_DATE_FORMAT, + isParsing = false) private def makeWriter(dataType: DataType): ValueWriter = dataType match { case NullType => diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala index c4f6121723491..e038f777c7a41 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala @@ -61,12 +61,13 @@ class JacksonParser( options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) private lazy val dateFormatter = DateFormatter( options.dateFormat, options.zoneId, options.locale, - legacyFormat = FAST_DATE_FORMAT) + legacyFormat = FAST_DATE_FORMAT, + isParsing = true) /** * Create a converter which converts the JSON documents held by the `JsonParser` diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala index 56b12784fd214..de396a4c63458 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JsonInferSchema.scala @@ -43,7 +43,7 @@ private[sql] class JsonInferSchema(options: JSONOptions) extends Serializable { options.zoneId, options.locale, legacyFormat = FAST_DATE_FORMAT, - needVarLengthSecondFraction = true) + isParsing = true) /** * Infer the type of a collection of json records in three stages: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala index 06e1cdc27e7d5..6d225ad9b7645 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateFormatter.scala @@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.util import java.text.SimpleDateFormat import java.time.{LocalDate, ZoneId} -import java.time.format.DateTimeFormatter import java.util.{Date, Locale} import org.apache.commons.lang3.time.FastDateFormat @@ -42,7 +41,8 @@ class Iso8601DateFormatter( pattern: String, zoneId: ZoneId, locale: Locale, - legacyFormat: LegacyDateFormats.LegacyDateFormat) + legacyFormat: LegacyDateFormats.LegacyDateFormat, + isParsing: Boolean) extends DateFormatter with DateTimeFormatterHelper { @transient @@ -125,12 +125,13 @@ object DateFormatter { format: Option[String], zoneId: ZoneId, locale: Locale = defaultLocale, - legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT): DateFormatter = { + legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT, + isParsing: Boolean = true): DateFormatter = { val pattern = format.getOrElse(defaultPattern) if (SQLConf.get.legacyTimeParserPolicy == LEGACY) { getLegacyFormatter(pattern, zoneId, locale, legacyFormat) } else { - val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat) + val df = new Iso8601DateFormatter(pattern, zoneId, locale, legacyFormat, isParsing) df.validatePatternString() df } @@ -153,8 +154,9 @@ object DateFormatter { format: String, zoneId: ZoneId, locale: Locale, - legacyFormat: LegacyDateFormat): DateFormatter = { - getFormatter(Some(format), zoneId, locale, legacyFormat) + legacyFormat: LegacyDateFormat, + isParsing: Boolean): DateFormatter = { + getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing) } def apply(format: String, zoneId: ZoneId): DateFormatter = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala index 5b9d8396530bc..ffa7cd438b29b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala @@ -89,9 +89,9 @@ trait DateTimeFormatterHelper { protected def getOrCreateFormatter( pattern: String, locale: Locale, - needVarLengthSecondFraction: Boolean = false): DateTimeFormatter = { - val newPattern = convertIncompatiblePattern(pattern) - val useVarLen = needVarLengthSecondFraction && newPattern.contains('S') + isParsing: Boolean = false): DateTimeFormatter = { + val newPattern = convertIncompatiblePattern(pattern, isParsing) + val useVarLen = isParsing && newPattern.contains('S') val key = (newPattern, locale, useVarLen) var formatter = cache.getIfPresent(key) if (formatter == null) { @@ -227,6 +227,12 @@ private object DateTimeFormatterHelper { formatter.format(LocalDate.of(2000, 1, 1)) == "1 1" } final val unsupportedLetters = Set('A', 'c', 'e', 'n', 'N', 'p') + // SPARK-31892: The week-based date fields are rarely used and really confusing for parsing values + // to datetime, especially when they are mixed with other non-week-based ones + // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can + // be directly resolved then the `q` do check for whether the month is valid, but if the date + // fields is incomplete, e.g. `yM`, the checking will be bypassed. + final val unsupportedLettersForParsing = Set('Y', 'W', 'w', 'E', 'u', 'F', 'q', 'Q') final val unsupportedPatternLengths = { // SPARK-31771: Disable Narrow-form TextStyle to avoid silent data change, as it is Full-form in // 2.4 @@ -246,7 +252,7 @@ private object DateTimeFormatterHelper { * @param pattern The input pattern. * @return The pattern for new parser */ - def convertIncompatiblePattern(pattern: String): String = { + def convertIncompatiblePattern(pattern: String, isParsing: Boolean = false): String = { val eraDesignatorContained = pattern.split("'").zipWithIndex.exists { case (patternPart, index) => // Text can be quoted using single quotes, we only check the non-quote parts. @@ -255,7 +261,8 @@ private object DateTimeFormatterHelper { (pattern + " ").split("'").zipWithIndex.map { case (patternPart, index) => if (index % 2 == 0) { - for (c <- patternPart if unsupportedLetters.contains(c)) { + for (c <- patternPart if unsupportedLetters.contains(c) || + (isParsing && unsupportedLettersForParsing.contains(c))) { throw new IllegalArgumentException(s"Illegal pattern character: $c") } for (style <- unsupportedPatternLengths if patternPart.contains(style)) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala index 3e302e2170390..97ecc430af4a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/TimestampFormatter.scala @@ -287,13 +287,13 @@ object TimestampFormatter { zoneId: ZoneId, locale: Locale = defaultLocale, legacyFormat: LegacyDateFormat = LENIENT_SIMPLE_DATE_FORMAT, - needVarLengthSecondFraction: Boolean = false): TimestampFormatter = { + isParsing: Boolean = false): TimestampFormatter = { val pattern = format.getOrElse(defaultPattern) if (SQLConf.get.legacyTimeParserPolicy == LEGACY) { getLegacyFormatter(pattern, zoneId, locale, legacyFormat) } else { val tf = new Iso8601TimestampFormatter( - pattern, zoneId, locale, legacyFormat, needVarLengthSecondFraction) + pattern, zoneId, locale, legacyFormat, isParsing) tf.validatePatternString() tf } @@ -319,23 +319,23 @@ object TimestampFormatter { zoneId: ZoneId, locale: Locale, legacyFormat: LegacyDateFormat, - needVarLengthSecondFraction: Boolean): TimestampFormatter = { - getFormatter(Some(format), zoneId, locale, legacyFormat, needVarLengthSecondFraction) + isParsing: Boolean): TimestampFormatter = { + getFormatter(Some(format), zoneId, locale, legacyFormat, isParsing) } def apply( format: String, zoneId: ZoneId, legacyFormat: LegacyDateFormat, - needVarLengthSecondFraction: Boolean): TimestampFormatter = { - getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, needVarLengthSecondFraction) + isParsing: Boolean): TimestampFormatter = { + getFormatter(Some(format), zoneId, defaultLocale, legacyFormat, isParsing) } def apply( format: String, zoneId: ZoneId, - needVarLengthSecondFraction: Boolean = false): TimestampFormatter = { - getFormatter(Some(format), zoneId, needVarLengthSecondFraction = needVarLengthSecondFraction) + isParsing: Boolean = false): TimestampFormatter = { + getFormatter(Some(format), zoneId, isParsing = isParsing) } def apply(zoneId: ZoneId): TimestampFormatter = { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala index 1ca7380ead413..c4c189c50febc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala @@ -1168,4 +1168,33 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkExceptionInExpression[ArithmeticException]( MillisToTimestamp(Literal(-92233720368547758L)), "long overflow") } + + test("Disable week-based date fields and quarter fields for parsing") { + + def checkSparkUpgrade(c: Char): Unit = { + checkExceptionInExpression[SparkUpgradeException]( + new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, "3.0") + checkExceptionInExpression[SparkUpgradeException]( + new ParseToDate(Literal("1"), Literal(c.toString)).child, "3.0") + checkExceptionInExpression[SparkUpgradeException]( + ToUnixTimestamp(Literal("1"), Literal(c.toString)), "3.0") + checkExceptionInExpression[SparkUpgradeException]( + UnixTimestamp(Literal("1"), Literal(c.toString)), "3.0") + } + + def checkNullify(c: Char): Unit = { + checkEvaluation(new ParseToTimestamp(Literal("1"), Literal(c.toString)).child, null) + checkEvaluation(new ParseToDate(Literal("1"), Literal(c.toString)).child, null) + checkEvaluation(ToUnixTimestamp(Literal("1"), Literal(c.toString)), null) + checkEvaluation(UnixTimestamp(Literal("1"), Literal(c.toString)), null) + } + + Seq('Y', 'W', 'w', 'E', 'u', 'F').foreach { l => + checkSparkUpgrade(l) + } + + Seq('q', 'Q').foreach { l => + checkNullify(l) + } + } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala index 18acaafc6d91a..c68bdacb13af7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelperSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.catalyst.util -import org.apache.spark.{SparkFunSuite, SparkUpgradeException} +import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._ class DateTimeFormatterHelperSuite extends SparkFunSuite { @@ -40,6 +40,13 @@ class DateTimeFormatterHelperSuite extends SparkFunSuite { val e = intercept[IllegalArgumentException](convertIncompatiblePattern(s"yyyy-MM-dd $l G")) assert(e.getMessage === s"Illegal pattern character: $l") } + unsupportedLettersForParsing.foreach { l => + val e = intercept[IllegalArgumentException] { + convertIncompatiblePattern(s"$l", isParsing = true) + } + assert(e.getMessage === s"Illegal pattern character: $l") + assert(convertIncompatiblePattern(s"$l").nonEmpty) + } unsupportedPatternLengths.foreach { style => val e1 = intercept[IllegalArgumentException] { convertIncompatiblePattern(s"yyyy-MM-dd $style") diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala index 7d503cc091172..22a1396d5efdc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/DateFormatterSuite.scala @@ -72,7 +72,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { DateFormatter.defaultPattern, getZoneId(timeZone), DateFormatter.defaultLocale, - legacyFormat) + legacyFormat, + isParsing = false) val days = formatter.parse(date) assert(date === formatter.format(days)) assert(date === formatter.format(daysToLocalDate(days))) @@ -106,7 +107,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { DateFormatter.defaultPattern, getZoneId(timeZone), DateFormatter.defaultLocale, - legacyFormat) + legacyFormat, + isParsing = false) val date = formatter.format(days) val parsed = formatter.parse(date) assert(days === parsed) @@ -173,7 +175,8 @@ class DateFormatterSuite extends SparkFunSuite with SQLHelper { DateFormatter.defaultPattern, getZoneId(timeZone), DateFormatter.defaultLocale, - legacyFormat) + legacyFormat, + isParsing = false) assert(LocalDate.ofEpochDay(formatter.parse("1000-01-01")) === LocalDate.of(1000, 1, 1)) assert(formatter.format(LocalDate.of(1000, 1, 1)) === "1000-01-01") assert(formatter.format(localDateToDays(LocalDate.of(1000, 1, 1))) === "1000-01-01") diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala index aeb238e75f02b..7ba45be06a5d3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/util/TimestampFormatterSuite.scala @@ -48,7 +48,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers val formatter = TimestampFormatter( "yyyy-MM-dd'T'HH:mm:ss.SSSSSS", getZoneId(zoneId), - needVarLengthSecondFraction = true) + isParsing = true) val microsSinceEpoch = formatter.parse(localDate) assert(microsSinceEpoch === expectedMicros(zoneId)) } @@ -73,7 +73,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers // Test only FAST_DATE_FORMAT because other legacy formats don't support formatting // in microsecond precision. LegacyDateFormats.FAST_DATE_FORMAT, - needVarLengthSecondFraction = false), + isParsing = false), TimestampFormatter.getFractionFormatter(getZoneId(zoneId))).foreach { formatter => val timestamp = formatter.format(microsSinceEpoch) assert(timestamp === expectedTimestamp(zoneId)) @@ -98,7 +98,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers outstandingZoneIds.foreach { zoneId => val timestamp = TimestampFormatter(pattern, zoneId).format(micros) val parsed = TimestampFormatter( - pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp) + pattern, zoneId, isParsing = true).parse(timestamp) assert(micros === parsed) } } @@ -119,7 +119,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers outstandingZoneIds.foreach { zoneId => val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSS" val micros = TimestampFormatter( - pattern, zoneId, needVarLengthSecondFraction = true).parse(timestamp) + pattern, zoneId, isParsing = true).parse(timestamp) val formatted = TimestampFormatter(pattern, zoneId).format(micros) assert(timestamp === formatted) } @@ -185,7 +185,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers test("parsing timestamp strings with various seconds fractions") { outstandingZoneIds.foreach { zoneId => def check(pattern: String, input: String, reference: String): Unit = { - val formatter = TimestampFormatter(pattern, zoneId, needVarLengthSecondFraction = true) + val formatter = TimestampFormatter(pattern, zoneId, isParsing = true) val expected = stringToTimestamp(UTF8String.fromString(reference), zoneId).get val actual = formatter.parse(input) assert(actual === expected) @@ -292,7 +292,7 @@ class TimestampFormatterSuite extends SparkFunSuite with SQLHelper with Matchers zoneId, TimestampFormatter.defaultLocale, legacyFormat, - needVarLengthSecondFraction = false) + isParsing = false) }.toSeq :+ TimestampFormatter.getFractionFormatter(zoneId) formatters.foreach { formatter => assert(microsToInstant(formatter.parse("1000-01-01 01:02:03")) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala index 292ac6db04baf..f7e225b0cdc96 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala @@ -134,7 +134,7 @@ object PartitioningUtils { val timestampFormatter = TimestampFormatter( timestampPartitionPattern, zoneId, - needVarLengthSecondFraction = true) + isParsing = true) // First, we need to parse every partition's path and see if we can find partition values. val (partitionValues, optDiscoveredBasePaths) = paths.map { path => parsePartition(path, typeInference, basePaths, userSpecifiedDataTypes, diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala index d3301ced2ba19..32a9558e91f10 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetPartitionDiscoverySuite.scala @@ -60,7 +60,7 @@ abstract class ParquetPartitionDiscoverySuite val timeZoneId = ZoneId.systemDefault() val df = DateFormatter(timeZoneId) val tf = TimestampFormatter( - timestampPartitionPattern, timeZoneId, needVarLengthSecondFraction = true) + timestampPartitionPattern, timeZoneId, isParsing = true) protected override def beforeAll(): Unit = { super.beforeAll()