Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ object FunctionRegistry {
expression[Log2]("log2"),
expression[Log]("ln"),
expression[Remainder]("mod", true),
expression[UnaryMinus]("negative"),
expression[UnaryMinus]("negative", true),
expression[Pi]("pi"),
expression[Pmod]("pmod"),
expression[UnaryPositive]("positive"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ case class TimeWindow(
override def dataType: DataType = new StructType()
.add(StructField("start", TimestampType))
.add(StructField("end", TimestampType))
override def prettyName: String = "window"

// This expression is replaced in the analyzer.
override lazy val resolved = false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ abstract class MaxMinBy extends DeclarativeAggregate {
def valueExpr: Expression
def orderingExpr: Expression

protected def funcName: String
// The predicate compares two ordering values.
protected def predicate(oldExpr: Expression, newExpr: Expression): Expression
// The arithmetic expression returns greatest/least value of all parameters.
Expand All @@ -46,7 +45,7 @@ abstract class MaxMinBy extends DeclarativeAggregate {
override def dataType: DataType = valueExpr.dataType

override def checkInputDataTypes(): TypeCheckResult =
TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $funcName")
TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $prettyName")

// The attributes used to keep extremum (max or min) and associated aggregated values.
private lazy val extremumOrdering =
Expand Down Expand Up @@ -101,7 +100,8 @@ abstract class MaxMinBy extends DeclarativeAggregate {
group = "agg_funcs",
since = "3.0.0")
case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
override protected def funcName: String = "max_by"

override def prettyName: String = "max_by"

override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
oldExpr > newExpr
Expand All @@ -120,7 +120,8 @@ case class MaxBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMin
group = "agg_funcs",
since = "3.0.0")
case class MinBy(valueExpr: Expression, orderingExpr: Expression) extends MaxMinBy {
override protected def funcName: String = "min_by"

override def prettyName: String = "min_by"

override protected def predicate(oldExpr: Expression, newExpr: Expression): Expression =
oldExpr < newExpr
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,12 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
case _ => numeric.negate(input)
}

override def sql: String = s"(- ${child.sql})"
override def sql: String = {
getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("-") match {
case "-" => s"(- ${child.sql})"
case funcName => s"$funcName(${child.sql})"
}
}
}

@ExpressionDescription(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,8 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI

override def toString: String = s"bit_count($child)"

override def prettyName: String = "bit_count"

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match {
case BooleanType => defineCodeGen(ctx, ev, c => s"if ($c) 1 else 0")
case _ => defineCodeGen(ctx, ev, c => s"java.lang.Long.bitCount($c)")
Expand All @@ -184,6 +186,4 @@ case class BitwiseCount(child: Expression) extends UnaryExpression with ExpectsI
case IntegerType => java.lang.Long.bitCount(input.asInstanceOf[Int])
case LongType => java.lang.Long.bitCount(input.asInstanceOf[Long])
}

override def sql: String = s"bit_count(${child.sql})"
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<!-- Automatically generated byExpressionsSchemaSuite -->
<!-- Automatically generated by ExpressionsSchemaSuite -->
## Summary
- Number of queries: 333
- Number of expressions that missing example: 34
Expand Down Expand Up @@ -278,7 +278,7 @@
| org.apache.spark.sql.catalyst.expressions.TruncTimestamp | date_trunc | SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359') | struct<date_trunc(YEAR, CAST(2015-03-05T09:32:05.359 AS TIMESTAMP)):timestamp> |
| org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
| org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<(- 1):int> |
| org.apache.spark.sql.catalyst.expressions.UnaryMinus | negative | SELECT negative(1) | struct<negative(1):int> |
| org.apache.spark.sql.catalyst.expressions.UnaryPositive | positive | N/A | N/A |
| org.apache.spark.sql.catalyst.expressions.Unhex | unhex | SELECT decode(unhex('537061726B2053514C'), 'UTF-8') | struct<decode(unhex(537061726B2053514C), UTF-8):string> |
| org.apache.spark.sql.catalyst.expressions.UnixTimestamp | unix_timestamp | SELECT unix_timestamp() | struct<unix_timestamp(current_timestamp(), yyyy-MM-dd HH:mm:ss):bigint> |
Expand Down Expand Up @@ -317,9 +317,9 @@
| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last_value | SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last_value(col, false):int> |
| org.apache.spark.sql.catalyst.expressions.aggregate.Last | last | SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col) | struct<last(col, false):int> |
| org.apache.spark.sql.catalyst.expressions.aggregate.Max | max | SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col) | struct<max(col):int> |
| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<maxby(x, y):string> |
| org.apache.spark.sql.catalyst.expressions.aggregate.MaxBy | max_by | SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<max_by(x, y):string> |
| org.apache.spark.sql.catalyst.expressions.aggregate.Min | min | SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col) | struct<min(col):int> |
| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<minby(x, y):string> |
| org.apache.spark.sql.catalyst.expressions.aggregate.MinBy | min_by | SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y) | struct<min_by(x, y):string> |
| org.apache.spark.sql.catalyst.expressions.aggregate.Percentile | percentile | SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col) | struct<percentile(col, CAST(0.3 AS DOUBLE), 1):double> |
| org.apache.spark.sql.catalyst.expressions.aggregate.Skewness | skewness | SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col) | struct<skewness(CAST(col AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.aggregate.StddevPop | stddev_pop | SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col) | struct<stddev_pop(CAST(col AS DOUBLE)):double> |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ struct<abs(-3.13):decimal(3,2),abs(CAST(-2.19 AS DOUBLE)):double>
-- !query
select positive('-1.11'), positive(-1.11), negative('-1.11'), negative(-1.11)
-- !query schema
struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),(- CAST(-1.11 AS DOUBLE)):double,(- -1.11):decimal(3,2)>
struct<(+ CAST(-1.11 AS DOUBLE)):double,(+ -1.11):decimal(3,2),negative(CAST(-1.11 AS DOUBLE)):double,negative(-1.11):decimal(3,2)>
-- !query output
-1.11 -1.11 1.11 1.11

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ class ExpressionsSchemaSuite extends QueryTest with SharedSparkSession {
}

val header = Seq(
s"<!-- Automatically generated by${getClass.getSimpleName} -->",
s"<!-- Automatically generated by ${getClass.getSimpleName} -->",
"## Summary",
s" - Number of queries: ${outputs.size}",
s" - Number of expressions that missing example: ${missingExamples.size}",
Expand Down