Skip to content

Commit b382267

Browse files
tarekbeckertarekbecker
committed
[SPARK-8199] fixed bug in day calculation; removed set TimeZone in HiveCompatibilitySuite for test purposes; removed Hive tests for second and minute, because we can cast '2015-03-18' to a timestamp and extract a minute/second from it
1 parent 1b2e540 commit b382267

File tree

4 files changed

+41
-25
lines changed

4 files changed

+41
-25
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeFunctions.scala‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -322,16 +322,16 @@ case class Day(child: Expression) extends DateFormatExpression with ImplicitCast
322322
dayInYear match {
323323
case i: Int if i <= 31 => i
324324
case i: Int if i <= 59 + leap => i - 31
325-
case i: Int if i <= 90 + leap => i - 59 + leap
326-
case i: Int if i <= 120 + leap => i - 90 + leap
327-
case i: Int if i <= 151 + leap => i - 120 + leap
328-
case i: Int if i <= 181 + leap => i - 151 + leap
329-
case i: Int if i <= 212 + leap => i - 181 + leap
330-
case i: Int if i <= 243 + leap => i - 212 + leap
331-
case i: Int if i <= 273 + leap => i - 243 + leap
332-
case i: Int if i <= 304 + leap => i - 273 + leap
333-
case i: Int if i <= 334 + leap => i - 304 + leap
334-
case i: Int => i - 334 + leap
325+
case i: Int if i <= 90 + leap => i - 59 - leap
326+
case i: Int if i <= 120 + leap => i - 90 - leap
327+
case i: Int if i <= 151 + leap => i - 120 - leap
328+
case i: Int if i <= 181 + leap => i - 151 - leap
329+
case i: Int if i <= 212 + leap => i - 181 - leap
330+
case i: Int if i <= 243 + leap => i - 212 - leap
331+
case i: Int if i <= 273 + leap => i - 243 - leap
332+
case i: Int if i <= 304 + leap => i - 273 - leap
333+
case i: Int if i <= 334 + leap => i - 304 - leap
334+
case i: Int => i - 334 - leap
335335
}
336336
}
337337

@@ -346,25 +346,25 @@ case class Day(child: Expression) extends DateFormatExpression with ImplicitCast
346346
} else if ($dayInYear <= 59 + $leap) {
347347
${ev.primitive} = $dayInYear - 31;
348348
} else if ($dayInYear <= 90 + $leap) {
349-
${ev.primitive} = $dayInYear - 59 + $leap;
349+
${ev.primitive} = $dayInYear - 59 - $leap;
350350
} else if ($dayInYear <= 120 + $leap) {
351-
${ev.primitive} = $dayInYear - 90 + $leap;
351+
${ev.primitive} = $dayInYear - 90 - $leap;
352352
} else if ($dayInYear <= 151 + $leap) {
353-
${ev.primitive} = $dayInYear - 120 + $leap;
353+
${ev.primitive} = $dayInYear - 120 - $leap;
354354
} else if ($dayInYear <= 181 + $leap) {
355-
${ev.primitive} = $dayInYear - 151 + $leap;
355+
${ev.primitive} = $dayInYear - 151 - $leap;
356356
} else if ($dayInYear <= 212 + $leap) {
357-
${ev.primitive} = $dayInYear - 181 + $leap;
357+
${ev.primitive} = $dayInYear - 181 - $leap;
358358
} else if ($dayInYear <= 243 + $leap) {
359-
${ev.primitive} = $dayInYear - 212 + $leap;
359+
${ev.primitive} = $dayInYear - 212 - $leap;
360360
} else if ($dayInYear <= 273 + $leap) {
361-
${ev.primitive} = $dayInYear - 243 + $leap;
361+
${ev.primitive} = $dayInYear - 243 - $leap;
362362
} else if ($dayInYear <= 304 + $leap) {
363-
${ev.primitive} = $dayInYear - 273 + $leap;
363+
${ev.primitive} = $dayInYear - 273 - $leap;
364364
} else if ($dayInYear <= 334 + $leap) {
365-
${ev.primitive} = $dayInYear - 304 + $leap;
365+
${ev.primitive} = $dayInYear - 304 - $leap;
366366
} else {
367-
${ev.primitive} = $dayInYear - 334 + $leap;
367+
${ev.primitive} = $dayInYear - 334 - $leap;
368368
}
369369
"""
370370
})

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DatetimeFunctionsSuite.scala‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,6 @@ class DatetimeFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
166166
}
167167
}
168168

169-
val sdf = new SimpleDateFormat("D")
170169
(1999 to 2000).foreach { y =>
171170
(0 to 11).foreach { m =>
172171
(0 to 5 * 24).foreach { i =>
@@ -185,6 +184,16 @@ class DatetimeFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
185184
checkEvaluation(Day(Cast(Literal(d), DateType)), 8)
186185
checkEvaluation(Day(Cast(Literal(sdfDate.format(d)), DateType)), 8)
187186
checkEvaluation(Day(Cast(Literal(ts), DateType)), 8)
187+
188+
(1999 to 2000).foreach { y =>
189+
val c = Calendar.getInstance()
190+
c.set(y, 0, 1, 0, 0, 0)
191+
(0 to 365).foreach { d =>
192+
c.add(Calendar.DATE, 1)
193+
checkEvaluation(Day(Cast(Literal(new Date(c.getTimeInMillis)), DateType)),
194+
c.get(Calendar.DAY_OF_MONTH))
195+
}
196+
}
188197
}
189198

190199
test("Seconds") {

‎sql/core/src/test/scala/org/apache/spark/sql/DatetimeExpressionsSuite.scala‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ class DatetimeExpressionsSuite extends QueryTest {
116116
checkAnswer(
117117
df.selectExpr("day(a)", "day(b)", "day(c)"),
118118
Row(8, 8, 8))
119+
120+
checkAnswer(
121+
df.selectExpr("day(CAST(\"2008-11-01 15:32:20\" AS DATE))"),
122+
Row(1)
123+
)
119124
}
120125

121126
test("dayInYear") {

‎sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,9 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
4343
override def beforeAll() {
4444
TestHive.cacheTables = true
4545
// Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
46-
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
46+
// TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
4747
// Add Locale setting
48-
Locale.setDefault(Locale.US)
48+
// Locale.setDefault(Locale.US)
4949
// Set a relatively small column batch size for testing purposes
5050
TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, 5)
5151
// Enable in-memory partition pruning for testing purposes
@@ -892,7 +892,8 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
892892
"udf_lpad",
893893
"udf_ltrim",
894894
"udf_map",
895-
"udf_minute",
895+
// "udf_minute", we can cast dates likes '2015-03-18' to a timestamp and extract the minutes.
896+
// Hive returns null for minute('2015-03-18')
896897
"udf_modulo",
897898
"udf_month",
898899
"udf_named_struct",
@@ -919,7 +920,8 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
919920
"udf_round_3",
920921
"udf_rpad",
921922
"udf_rtrim",
922-
"udf_second",
923+
// "udf_second", we can cast dates likes '2015-03-18' to a timestamp and extract the seconds.
924+
// Hive returns null for second('2015-03-18')
923925
"udf_sign",
924926
"udf_sin",
925927
"udf_smallint",

0 commit comments

Comments
 (0)