Skip to content

Commit d0e2f99

Browse files
tarekbeckertarekbecker
committed
date functions
1 parent a458efc commit d0e2f99

File tree

5 files changed

+590
-1
lines changed

5 files changed

+590
-1
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,17 @@ object FunctionRegistry {
152152
expression[Substring]("substr"),
153153
expression[Substring]("substring"),
154154
expression[Upper]("ucase"),
155-
expression[Upper]("upper")
155+
expression[Upper]("upper"),
156+
157+
// datetime functions
158+
expression[DateFormat]("dateformat"),
159+
expression[Year]("year"),
160+
expression[Month]("month"),
161+
expression[Day]("day"),
162+
expression[Hour]("hour"),
163+
expression[Minute]("minute"),
164+
expression[Second]("second"),
165+
expression[WeekOfYear]("weekofyear")
156166
)
157167

158168
val builtin: FunctionRegistry = {

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetime.scala‎

Lines changed: 229 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.catalyst.expressions
19+
20+
import java.sql.Date
21+
import java.text.SimpleDateFormat
22+
23+
import org.apache.spark.sql.catalyst.expressions.codegen.{GeneratedExpressionCode, CodeGenContext}
24+
import org.apache.spark.sql.catalyst.util.DateTimeUtils
25+
import org.apache.spark.sql.types._
26+
import org.apache.spark.unsafe.types.UTF8String
27+
28+
case class DateFormat(left: Expression, right: Expression)
29+
extends BinaryExpression with ExpectsInputTypes {
30+
31+
override def dataType: DataType = StringType
32+
33+
override def expectedChildTypes: Seq[DataType] = Seq(TimestampType, StringType)
34+
35+
override def foldable: Boolean = left.foldable && right.foldable
36+
37+
override def nullable: Boolean = true
38+
39+
override def eval(input: InternalRow): Any = {
40+
val valueLeft = left.eval(input)
41+
if (valueLeft == null) {
42+
null
43+
} else {
44+
val valueRight = right.eval(input)
45+
if (valueRight == null) {
46+
null
47+
} else {
48+
val sdf = new SimpleDateFormat(valueRight.asInstanceOf[UTF8String].toString)
49+
left.dataType match {
50+
case TimestampType =>
51+
UTF8String.fromString(sdf.format(new Date(valueLeft.asInstanceOf[Long] / 10000)))
52+
case DateType =>
53+
UTF8String.fromString(sdf.format(DateTimeUtils.toJavaDate(valueLeft.asInstanceOf[Int])))
54+
case StringType =>
55+
UTF8String.fromString(
56+
sdf.format(DateTimeUtils.stringToTime(valueLeft.asInstanceOf[UTF8String].toString)))
57+
}
58+
}
59+
}
60+
}
61+
62+
override def toString: String = s"DateFormat($left, $right)"
63+
64+
override protected def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
65+
val sdf = "java.text.SimpleDateFormat"
66+
val utf8 = "org.apache.spark.unsafe.types.UTF8String"
67+
val dtUtils = "org.apache.spark.sql.catalyst.util.DateTimeUtils"
68+
69+
val eval1 = left.gen(ctx)
70+
val eval2 = right.gen(ctx)
71+
72+
val calc = left.dataType match {
73+
case TimestampType =>
74+
s"""$utf8.fromString(sdf.format(new java.sql.Date(${eval1.primitive} / 10000)));"""
75+
case DateType =>
76+
s"""$utf8.fromString(
77+
sdf.format($dtUtils.toJavaDate(${eval1.primitive})));"""
78+
case StringType =>
79+
s"""
80+
$utf8.fromString(sdf.format(new java.sql.Date($dtUtils.stringToTime(${eval1.primitive}.toString()).getTime())));
81+
"""
82+
}
83+
84+
s"""
85+
${eval1.code}
86+
boolean ${ev.isNull} = ${eval1.isNull};
87+
${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)};
88+
if (!${ev.isNull}) {
89+
${eval2.code}
90+
if (!${eval2.isNull}) {
91+
$sdf sdf = new $sdf(${eval2.primitive}.toString());
92+
${ev.primitive} = $calc
93+
} else {
94+
${ev.isNull} = true;
95+
}
96+
}
97+
"""
98+
}
99+
}
100+
101+
case class Year(child: Expression) extends UnaryExpression with ExpectsInputTypes {
102+
103+
override def dataType: DataType = IntegerType
104+
105+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
106+
107+
override def foldable: Boolean = child.foldable
108+
109+
override def nullable: Boolean = true
110+
111+
override def eval(input: InternalRow): Any = {
112+
DateFormat(child, Literal("y")).eval(input) match {
113+
case null => null
114+
case x: UTF8String => x.toString.toInt
115+
}
116+
}
117+
118+
}
119+
120+
case class Month(child: Expression) extends UnaryExpression with ExpectsInputTypes {
121+
122+
override def dataType: DataType = IntegerType
123+
124+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
125+
126+
override def foldable: Boolean = child.foldable
127+
128+
override def nullable: Boolean = true
129+
130+
override def eval(input: InternalRow): Any = {
131+
DateFormat(child, Literal("M")).eval(input) match {
132+
case null => null
133+
case x: UTF8String => x.toString.toInt
134+
}
135+
}
136+
}
137+
138+
case class Day(child: Expression) extends UnaryExpression with ExpectsInputTypes {
139+
140+
override def dataType: DataType = IntegerType
141+
142+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
143+
144+
override def foldable: Boolean = child.foldable
145+
146+
override def nullable: Boolean = true
147+
148+
override def eval(input: InternalRow): Any = {
149+
DateFormat(child, Literal("d")).eval(input) match {
150+
case null => null
151+
case x: UTF8String => x.toString.toInt
152+
}
153+
}
154+
155+
}
156+
157+
case class Hour(child: Expression) extends UnaryExpression with ExpectsInputTypes {
158+
159+
override def dataType: DataType = IntegerType
160+
161+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
162+
163+
override def foldable: Boolean = child.foldable
164+
165+
override def nullable: Boolean = true
166+
167+
override def eval(input: InternalRow): Any = {
168+
DateFormat(child, Literal("H")).eval(input) match {
169+
case null => null
170+
case x: UTF8String => x.toString.toInt
171+
}
172+
}
173+
}
174+
175+
case class Minute(child: Expression) extends UnaryExpression with ExpectsInputTypes {
176+
177+
override def dataType: DataType = IntegerType
178+
179+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
180+
181+
override def foldable: Boolean = child.foldable
182+
183+
override def nullable: Boolean = true
184+
185+
override def eval(input: InternalRow): Any = {
186+
DateFormat(child, Literal("m")).eval(input) match {
187+
case null => null
188+
case x: UTF8String => x.toString.toInt
189+
}
190+
}
191+
}
192+
193+
case class Second(child: Expression) extends UnaryExpression with ExpectsInputTypes {
194+
195+
override def dataType: DataType = IntegerType
196+
197+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
198+
199+
override def foldable: Boolean = child.foldable
200+
201+
override def nullable: Boolean = true
202+
203+
override def eval(input: InternalRow): Any = {
204+
DateFormat(child, Literal("s")).eval(input) match {
205+
case null => null
206+
case x: UTF8String => x.toString.toInt
207+
}
208+
}
209+
}
210+
211+
case class WeekOfYear(child: Expression) extends UnaryExpression with ExpectsInputTypes {
212+
213+
override def dataType: DataType = IntegerType
214+
215+
override def expectedChildTypes: Seq[DataType] = Seq(DateType, StringType, TimestampType)
216+
217+
override def foldable: Boolean = child.foldable
218+
219+
override def nullable: Boolean = true
220+
221+
override def eval(input: InternalRow): Any = {
222+
DateFormat(child, Literal("w")).eval(input) match {
223+
case null => null
224+
case x: UTF8String => x.toString.toInt
225+
}
226+
}
227+
228+
229+
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateTimeFunctionsSuite.scala‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.sql.catalyst.expressions
19+
20+
import java.sql.{Timestamp, Date}
21+
import java.text.SimpleDateFormat
22+
23+
import org.apache.spark.SparkFunSuite
24+
25+
class DateTimeFunctionsSuite extends SparkFunSuite with ExpressionEvalHelper {
26+
27+
val sdf = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss")
28+
val d = new Date(sdf.parse("2015/04/08 13:10:15").getTime)
29+
val ts = new Timestamp(sdf.parse("2013/04/08 13:10:15").getTime)
30+
31+
test("DateFormat") {
32+
checkEvaluation(DateFormat(Literal(d), Literal("y")), "2015")
33+
checkEvaluation(DateFormat(Literal(d.toString), Literal("y")), "2015")
34+
checkEvaluation(DateFormat(Literal(ts), Literal("y")), "2013")
35+
}
36+
37+
test("Year") {
38+
checkEvaluation(Year(Literal(d)), 2015)
39+
checkEvaluation(Year(Literal(d.toString)), 2015)
40+
checkEvaluation(Year(Literal(ts)), 2013)
41+
}
42+
43+
test("Month") {
44+
checkEvaluation(Month(Literal(d)), 4)
45+
checkEvaluation(Month(Literal(d.toString)), 4)
46+
checkEvaluation(Month(Literal(ts)), 4)
47+
}
48+
49+
test("Day") {
50+
checkEvaluation(Day(Literal(d)), 8)
51+
checkEvaluation(Day(Literal(d.toString)), 8)
52+
checkEvaluation(Day(Literal(ts)), 8)
53+
}
54+
55+
test("Hour") {
56+
checkEvaluation(Hour(Literal(d)), 0)
57+
checkEvaluation(Hour(Literal(d.toString)), 0)
58+
checkEvaluation(Hour(Literal(ts)), 13)
59+
}
60+
61+
test("Minute") {
62+
checkEvaluation(Minute(Literal(d)), 0)
63+
checkEvaluation(Minute(Literal(d.toString)), 0)
64+
checkEvaluation(Minute(Literal(ts)), 10)
65+
}
66+
67+
test("Seconds") {
68+
checkEvaluation(Second(Literal(d)), 0)
69+
checkEvaluation(Second(Literal(d.toString)), 0)
70+
checkEvaluation(Second(Literal(ts)), 15)
71+
}
72+
73+
test("WeekOfYear") {
74+
checkEvaluation(WeekOfYear(Literal(d)), 15)
75+
checkEvaluation(WeekOfYear(Literal(d.toString)), 15)
76+
checkEvaluation(WeekOfYear(Literal(ts)), 15)
77+
}
78+
79+
}

0 commit comments

Comments
 (0)