Skip to content

Commit 3ee0c05

Browse files
brkyvzbrkyvz
committed
added python functions
1 parent b14cd23 commit 3ee0c05

File tree

4 files changed

+190
-402
lines changed

4 files changed

+190
-402
lines changed

‎python/pyspark/sql/functions.py‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,22 @@
3232

3333
__all__ = ['countDistinct', 'approxCountDistinct', 'udf']
3434

35+
def _function_obj(sc, is_math=False):
36+
if not is_math:
37+
return sc._jvm.functions
38+
else:
39+
return sc._jvm.mathfunctions
3540

36-
def _create_function(name, doc=""):
41+
def _create_function(name, doc="", is_math=False):
3742
""" Create a function for aggregator by name"""
3843
def _(col):
3944
sc = SparkContext._active_spark_context
40-
jc = getattr(sc._jvm.functions, name)(col._jc if isinstance(col, Column) else col)
45+
jc = getattr(_function_obj(sc, is_math), name)(col._jc if isinstance(col, Column) else col)
4146
return Column(jc)
4247
_.__name__ = name
4348
_.__doc__ = doc
4449
return _
4550

46-
4751
_functions = {
4852
'lit': 'Creates a :class:`Column` of literal value.',
4953
'col': 'Returns a :class:`Column` based on the given column name.',
@@ -54,7 +58,7 @@ def _(col):
5458
'upper': 'Converts a string expression to upper case.',
5559
'lower': 'Converts a string expression to upper case.',
5660
'sqrt': 'Computes the square root of the specified float value.',
57-
'abs': 'Computes the absolutle value.',
61+
'abs': 'Computes the absolute value.',
5862

5963
'max': 'Aggregate function: returns the maximum value of the expression in a group.',
6064
'min': 'Aggregate function: returns the minimum value of the expression in a group.',
@@ -67,11 +71,48 @@ def _(col):
6771
'sumDistinct': 'Aggregate function: returns the sum of distinct values in the expression.',
6872
}
6973

74+
# math functions are found under another object therefore, they need to be handled separately
75+
_math_functions = {
76+
'acos': 'Computes the cosine inverse of the given value; the returned angle is in the range' + \
77+
'0.0 through pi.',
78+
'asin': 'Computes the sine inverse of the given value; the returned angle is in the range' + \
79+
'-pi/2 through pi/2.',
80+
'atan': 'Computes the tangent inverse of the given value.',
81+
'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x, y) to' + \
82+
'polar coordinates (r, theta).',
83+
'cbrt': 'Computes the cube-root of the given value.',
84+
'ceil': 'Computes the ceiling of the given value.',
85+
'cos': 'Computes the cosine of the given value.',
86+
'cosh': 'Computes the hyperbolic cosine of the given value.',
87+
'exp': 'Computes the exponential of the given value.',
88+
'expm1': 'Computes the exponential of the given value minus one.',
89+
'floor': 'Computes the floor of the given value.',
90+
'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
91+
'log': 'Computes the natural logarithm of the given value.',
92+
'log10': 'Computes the logarithm of the given value in Base 10.',
93+
'log1p': 'Computes the natural logarithm of the given value plus one.',
94+
'pow': 'Returns the value of the first argument raised to the power of the second argument.',
95+
'rint': 'Returns the double value that is closest in value to the argument and' + \
96+
' is equal to a mathematical integer.',
97+
'signum': 'Computes the signum of the given value.',
98+
'sin': 'Computes the sine of the given value.',
99+
'sinh': 'Computes the hyperbolic sine of the given value.',
100+
'tan': 'Computes the tangent of the given value.',
101+
'tanh': 'Computes the hyperbolic tangent of the given value.',
102+
'toDeg': 'Converts an angle measured in radians to an approximately equivalent angle ' + \
103+
'measured in degrees.',
104+
'toRad': 'Converts an angle measured in degrees to an approximately equivalent angle ' + \
105+
'measured in radians.'
106+
}
107+
70108

71109
for _name, _doc in _functions.items():
72110
globals()[_name] = _create_function(_name, _doc)
111+
for _name, _doc in _math_functions.items():
112+
globals()[_name] = _create_function(_name, _doc, True)
73113
del _name, _doc
74114
__all__ += _functions.keys()
115+
__all__ += _math_functions.keys()
75116
__all__.sort()
76117

77118

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,6 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
6565
}
6666
}
6767

68-
case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER")
69-
70-
case class Hypot(
71-
left: Expression,
72-
right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT")
73-
7468
case class Atan2(
7569
left: Expression,
7670
right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") {
@@ -91,3 +85,9 @@ case class Atan2(
9185
}
9286
}
9387
}
88+
89+
case class Hypot(
90+
left: Expression,
91+
right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT")
92+
93+
case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER")

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala‎

Lines changed: 23 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,16 @@ import org.apache.spark.sql.types._
2525
* input format, therefore these functions extend `ExpectsInputTypes`.
2626
* @param name The short name of the function
2727
*/
28-
abstract class MathematicalExpression(name: String)
28+
abstract class MathematicalExpression(f: Double => Double, name: String)
2929
extends UnaryExpression with Serializable with ExpectsInputTypes {
3030
self: Product =>
3131
type EvaluatedType = Any
3232

33+
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
3334
override def dataType: DataType = DoubleType
3435
override def foldable: Boolean = child.foldable
3536
override def nullable: Boolean = true
3637
override def toString: String = s"$name($child)"
37-
}
38-
39-
/**
40-
* A unary expression specifically for math functions that take a `Double` as input and return
41-
* a `Double`.
42-
* @param f The math function.
43-
* @param name The short name of the function
44-
*/
45-
abstract class MathematicalExpressionForDouble(f: Double => Double, name: String)
46-
extends MathematicalExpression(name) { self: Product =>
47-
48-
override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
4938

5039
override def eval(input: Row): Any = {
5140
val evalE = child.eval(input)
@@ -58,111 +47,46 @@ abstract class MathematicalExpressionForDouble(f: Double => Double, name: String
5847
}
5948
}
6049

61-
/**
62-
* A unary expression specifically for math functions that take an `Int` as input and return
63-
* an `Int`.
64-
* @param f The math function.
65-
* @param name The short name of the function
66-
*/
67-
abstract class MathematicalExpressionForInt(f: Int => Int, name: String)
68-
extends MathematicalExpression(name) { self: Product =>
50+
case class Acos(child: Expression) extends MathematicalExpression(math.acos, "ACOS")
6951

70-
override def dataType: DataType = IntegerType
71-
override def expectedChildTypes: Seq[DataType] = Seq(IntegerType)
52+
case class Asin(child: Expression) extends MathematicalExpression(math.asin, "ASIN")
7253

73-
override def eval(input: Row): Any = {
74-
val evalE = child.eval(input)
75-
if (evalE == null) null else f(evalE.asInstanceOf[Int])
76-
}
77-
}
54+
case class Atan(child: Expression) extends MathematicalExpression(math.atan, "ATAN")
7855

79-
/**
80-
* A unary expression specifically for math functions that take a `Float` as input and return
81-
* a `Float`.
82-
* @param f The math function.
83-
* @param name The short name of the function
84-
*/
85-
abstract class MathematicalExpressionForFloat(f: Float => Float, name: String)
86-
extends MathematicalExpression(name) { self: Product =>
56+
case class Cbrt(child: Expression) extends MathematicalExpression(math.cbrt, "CBRT")
8757

88-
override def dataType: DataType = FloatType
89-
override def expectedChildTypes: Seq[DataType] = Seq(FloatType)
58+
case class Ceil(child: Expression) extends MathematicalExpression(math.ceil, "CEIL")
9059

91-
override def eval(input: Row): Any = {
92-
val evalE = child.eval(input)
93-
if (evalE == null) {
94-
null
95-
} else {
96-
val result = f(evalE.asInstanceOf[Float])
97-
if (result.isNaN) null else result
98-
}
99-
}
100-
}
101-
102-
/**
103-
* A unary expression specifically for math functions that take a `Long` as input and return
104-
* a `Long`.
105-
* @param f The math function.
106-
* @param name The short name of the function
107-
*/
108-
abstract class MathematicalExpressionForLong(f: Long => Long, name: String)
109-
extends MathematicalExpression(name) { self: Product =>
110-
111-
override def dataType: DataType = LongType
112-
override def expectedChildTypes: Seq[DataType] = Seq(LongType)
113-
114-
override def eval(input: Row): Any = {
115-
val evalE = child.eval(input)
116-
if (evalE == null) null else f(evalE.asInstanceOf[Long])
117-
}
118-
}
119-
120-
case class Sin(child: Expression) extends MathematicalExpressionForDouble(math.sin, "SIN")
121-
122-
case class Asin(child: Expression) extends MathematicalExpressionForDouble(math.asin, "ASIN")
123-
124-
case class Sinh(child: Expression) extends MathematicalExpressionForDouble(math.sinh, "SINH")
125-
126-
case class Cos(child: Expression) extends MathematicalExpressionForDouble(math.cos, "COS")
60+
case class Cos(child: Expression) extends MathematicalExpression(math.cos, "COS")
12761

128-
case class Acos(child: Expression) extends MathematicalExpressionForDouble(math.acos, "ACOS")
62+
case class Cosh(child: Expression) extends MathematicalExpression(math.cosh, "COSH")
12963

130-
case class Cosh(child: Expression) extends MathematicalExpressionForDouble(math.cosh, "COSH")
64+
case class Exp(child: Expression) extends MathematicalExpression(math.exp, "EXP")
13165

132-
case class Tan(child: Expression) extends MathematicalExpressionForDouble(math.tan, "TAN")
66+
case class Expm1(child: Expression) extends MathematicalExpression(math.expm1, "EXPM1")
13367

134-
case class Atan(child: Expression) extends MathematicalExpressionForDouble(math.atan, "ATAN")
68+
case class Floor(child: Expression) extends MathematicalExpression(math.floor, "FLOOR")
13569

136-
case class Tanh(child: Expression) extends MathematicalExpressionForDouble(math.tanh, "TANH")
70+
case class Log(child: Expression) extends MathematicalExpression(math.log, "LOG")
13771

138-
case class Ceil(child: Expression) extends MathematicalExpressionForDouble(math.ceil, "CEIL")
72+
case class Log10(child: Expression) extends MathematicalExpression(math.log10, "LOG10")
13973

140-
case class Floor(child: Expression) extends MathematicalExpressionForDouble(math.floor, "FLOOR")
74+
case class Log1p(child: Expression) extends MathematicalExpression(math.log1p, "LOG1P")
14175

142-
case class Rint(child: Expression) extends MathematicalExpressionForDouble(math.rint, "ROUND")
76+
case class Rint(child: Expression) extends MathematicalExpression(math.rint, "ROUND")
14377

144-
case class Cbrt(child: Expression) extends MathematicalExpressionForDouble(math.cbrt, "CBRT")
78+
case class Signum(child: Expression) extends MathematicalExpression(math.signum, "SIGNUM")
14579

146-
case class Signum(child: Expression) extends MathematicalExpressionForDouble(math.signum, "SIGNUM")
80+
case class Sin(child: Expression) extends MathematicalExpression(math.sin, "SIN")
14781

148-
case class ISignum(child: Expression) extends MathematicalExpressionForInt(math.signum, "ISIGNUM")
82+
case class Sinh(child: Expression) extends MathematicalExpression(math.sinh, "SINH")
14983

150-
case class FSignum(child: Expression) extends MathematicalExpressionForFloat(math.signum, "FSIGNUM")
84+
case class Tan(child: Expression) extends MathematicalExpression(math.tan, "TAN")
15185

152-
case class LSignum(child: Expression) extends MathematicalExpressionForLong(math.signum, "LSIGNUM")
86+
case class Tanh(child: Expression) extends MathematicalExpression(math.tanh, "TANH")
15387

15488
case class ToDegrees(child: Expression)
155-
extends MathematicalExpressionForDouble(math.toDegrees, "DEGREES")
89+
extends MathematicalExpression(math.toDegrees, "DEGREES")
15690

15791
case class ToRadians(child: Expression)
158-
extends MathematicalExpressionForDouble(math.toRadians, "RADIANS")
159-
160-
case class Log(child: Expression) extends MathematicalExpressionForDouble(math.log, "LOG")
161-
162-
case class Log10(child: Expression) extends MathematicalExpressionForDouble(math.log10, "LOG10")
163-
164-
case class Log1p(child: Expression) extends MathematicalExpressionForDouble(math.log1p, "LOG1P")
165-
166-
case class Exp(child: Expression) extends MathematicalExpressionForDouble(math.exp, "EXP")
167-
168-
case class Expm1(child: Expression) extends MathematicalExpressionForDouble(math.expm1, "EXPM1")
92+
extends MathematicalExpression(math.toRadians, "RADIANS")

0 commit comments

Comments
 (0)