From 1129a20128d34a5e3dd9af840ee119b977c050b3 Mon Sep 17 00:00:00 2001 From: xinyunh Date: Thu, 14 Aug 2014 13:13:02 -0700 Subject: [PATCH 1/7] add 'Last' component --- .../apache/spark/sql/catalyst/SqlParser.scala | 2 ++ .../spark/sql/catalyst/dsl/package.scala | 1 + .../sql/catalyst/expressions/aggregates.scala | 29 +++++++++++++++++++ 3 files changed, 32 insertions(+) mode change 100644 => 100755 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala mode change 100644 => 100755 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala mode change 100644 => 100755 sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala old mode 100644 new mode 100755 index a88bd859fc85e..8864780456c76 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -81,6 +81,7 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val DISTINCT = Keyword("DISTINCT") protected val FALSE = Keyword("FALSE") protected val FIRST = Keyword("FIRST") + protected val LAST = Keyword("LAST") protected val FROM = Keyword("FROM") protected val FULL = Keyword("FULL") protected val GROUP = Keyword("GROUP") @@ -311,6 +312,7 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ _ ~ _ ~ _ ~ _ ~ e => ApproxCountDistinct(e, s.toDouble) } | FIRST ~> "(" ~> expression <~ ")" ^^ { case exp => First(exp) } | + LAST ~> "(" ~> expression <~ ")" ^^ { case exp => Last(exp) } | AVG ~> "(" ~> expression <~ ")" ^^ { case exp => Average(exp) } | MIN ~> "(" ~> expression <~ ")" ^^ { case exp => Min(exp) } | MAX ~> "(" ~> expression <~ ")" ^^ { case exp => Max(exp) } | diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala old mode 100644 new mode 100755 index f44521d6381c9..deb622c39faf5 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -132,6 +132,7 @@ package object dsl { def approxCountDistinct(e: Expression, rsd: Double = 0.05) = ApproxCountDistinct(e, rsd) def avg(e: Expression) = Average(e) def first(e: Expression) = First(e) + def last(e: Expression) = Last(e) def min(e: Expression) = Min(e) def max(e: Expression) = Max(e) def upper(e: Expression) = Upper(e) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala old mode 100644 new mode 100755 index 15560a2a933ad..47338a2e92eac --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala @@ -344,6 +344,21 @@ case class First(child: Expression) extends PartialAggregate with trees.UnaryNod override def newInstance() = new FirstFunction(child, this) } +case class Last(child: Expression) extends PartialAggregate with trees.UnaryNode[Expression] { + override def references = child.references + override def nullable = true + override def dataType = child.dataType + override def toString = s"LAST($child)" + + override def asPartial: SplitEvaluation = { + val partialLast = Alias(Last(child), "PartialLast")() + SplitEvaluation( + Last(partialLast.toAttribute), + partialLast :: Nil) + } + override def newInstance() = new LastFunction(child, this) +} + case class AverageFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { @@ -489,3 +504,17 @@ case class FirstFunction(expr: Expression, base: AggregateExpression) extends Ag override def eval(input: Row): Any = result } + +case class LastFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { + def this() = this(null, null) // Required for serialization. + + var result: Any = null + + override def update(input: Row): Unit = { + if (result == null) { + result = input + } + } + + override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) else null +} From b285818d1532b2ff103f048c4172b448e6bda4bc Mon Sep 17 00:00:00 2001 From: xinyunh Date: Thu, 14 Aug 2014 16:10:28 -0700 Subject: [PATCH 2/7] fix the bug in 'Last' component --- .../apache/spark/sql/catalyst/expressions/aggregates.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala index 47338a2e92eac..f519d96963e1c 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala @@ -511,9 +511,7 @@ case class LastFunction(expr: Expression, base: AggregateExpression) extends Agg var result: Any = null override def update(input: Row): Unit = { - if (result == null) { - result = input - } + result = input } override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) else null From 1a24fcd6c22bb195fb68ca335610f227c35b4a3c Mon Sep 17 00:00:00 2001 From: bomeng Date: Wed, 20 Aug 2014 11:16:35 -0700 Subject: [PATCH 3/7] add abs() function support --- .../apache/spark/sql/catalyst/SqlParser.scala | 2 + .../sql/catalyst/expressions/arithmetic.scala | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index 8864780456c76..0eff1614c3c95 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -125,6 +125,7 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val SUBSTR = Keyword("SUBSTR") protected val SUBSTRING = Keyword("SUBSTRING") protected val SQRT = Keyword("SQRT") + protected val ABS = Keyword("ABS") // Use reflection to find the reserved words defined in this class. protected val reservedWords = @@ -328,6 +329,7 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ "," ~ p ~ "," ~ l => Substring(s,p,l) } | SQRT ~> "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) } | + ABS ~> "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) } | ident ~ "(" ~ repsep(expression, ",") <~ ")" ^^ { case udfName ~ _ ~ exprs => UnresolvedFunction(udfName, exprs) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index f988fb010b107..25d99eb4aee14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.catalyst.analysis.UnresolvedException import org.apache.spark.sql.catalyst.types._ +import scala.math.pow case class UnaryMinus(child: Expression) extends UnaryExpression { type EvaluatedType = Any @@ -129,3 +130,52 @@ case class MaxOf(left: Expression, right: Expression) extends Expression { override def toString = s"MaxOf($left, $right)" } + +/** + * A function that get the absolute value of the numeric value. + */ +case class Abs(child: Expression) extends UnaryExpression { + type EvaluatedType = Any + + def dataType = child.dataType + override def foldable = child.foldable + def nullable = child.nullable + override def toString = s"Abs($child)" + + override def eval(input: Row): Any = n1(child, input, _.abs(_)) +} + +/** + * A function that get the power value of two parameters. + * First one is taken as base while second one taken as exponent + */ +case class Power(base: Expression, exponent: Expression) extends Expression { + type EvaluatedType = Any + + def dataType: DataType = { + if (!resolved) throw new UnresolvedException(this, s"Cannot resolve since $children are not resolved") + DoubleType + } + override def foldable = base.foldable && exponent.foldable + def nullable: Boolean = base.nullable || exponent.nullable + override def toString = s"Power($base, $exponent)" + + override def children = base :: exponent :: Nil + + override def eval(input: Row): Any = { + def convertToDouble(num: EvaluatedType): Double = { + num match { + case d:Double => d + case i:Integer => i.doubleValue() + case f:Float => f.toDouble + } + } + + val base_v = base.eval(input) + val exponent_v = exponent.eval(input) + + if ((base_v == null) || (exponent_v == null)) null + else pow(convertToDouble(base_v), convertToDouble(exponent_v)) + } + +} From 75c1d5da0081ced06927d47df5595c17a4cb5a82 Mon Sep 17 00:00:00 2001 From: bomeng Date: Wed, 20 Aug 2014 11:35:48 -0700 Subject: [PATCH 4/7] Modify the code of POWER and ABS. Move them to the file arithmetic --- .../apache/spark/sql/catalyst/SqlParser.scala | 5 ++ .../org/apache/spark/sql/SQLQuerySuite.scala | 56 ++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index 0eff1614c3c95..c10445c0f691a 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -126,6 +126,8 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val SUBSTRING = Keyword("SUBSTRING") protected val SQRT = Keyword("SQRT") protected val ABS = Keyword("ABS") + protected val POW = Keyword("POW") + protected val POWER = Keyword("POWER") // Use reflection to find the reserved words defined in this class. protected val reservedWords = @@ -330,6 +332,9 @@ class SqlParser extends StandardTokenParsers with PackratParsers { } | SQRT ~> "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) } | ABS ~> "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) } | + (POW | POWER) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ { + case s ~ "," ~ p => Power(s,p) + } | ident ~ "(" ~ repsep(expression, ",") <~ ")" ^^ { case udfName ~ _ ~ exprs => UnresolvedFunction(udfName, exprs) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 1ac205937714c..4a6842c66ebd7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -41,6 +41,58 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { } + test("SPARK-3176 Added Parser of SQL ABS()") { + checkAnswer( + sql("SELECT ABS(-1.3)"), + 1.3) + checkAnswer( + sql("SELECT ABS(0.0)"), + 0.0) + checkAnswer( + sql("SELECT ABS(2.5)"), + 2.5) + } + + test("SPARK-3176 Added Parser of SQL POWER()") { + checkAnswer( + sql("SELECT POWER(0, 512.0)"), + 0.0) + checkAnswer( + sql("SELECT POW(1.0, 256.0)"), + 1.0) + checkAnswer( + sql("SELECT POWER(1, -128)"), + 1.0) + checkAnswer( + sql("SELECT POW(-1.0, -63)"), + -1.0) + checkAnswer( + sql("SELECT POWER(-1, 32.0)"), + 1.0) + checkAnswer( + sql("SELECT POW(2, 8)"), + 256.0) + checkAnswer( + sql("SELECT POWER(0.5, 2)"), + 0.25) + checkAnswer( + sql("SELECT POW(2, -2)"), + 0.25) + checkAnswer( + sql("SELECT POWER(8, 1)"), + 8.0) + checkAnswer( + sql("SELECT POW(16, 0.5)"), + 4.0) + } + + test("SPARK-3176 Added Parser of SQL LAST()") { + checkAnswer( + sql("SELECT LAST(n) FROM lowerCaseData"), + 4) + } + + test("SPARK-2041 column name equals tablename") { checkAnswer( sql("SELECT tableName FROM tableName"), @@ -53,14 +105,14 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { (1 to 100).map(x => Row(math.sqrt(x.toDouble))).toSeq ) } - + test("SQRT with automatic string casts") { checkAnswer( sql("SELECT SQRT(CAST(key AS STRING)) FROM testData"), (1 to 100).map(x => Row(math.sqrt(x.toDouble))).toSeq ) } - + test("SPARK-2407 Added Parser of SQL SUBSTR()") { checkAnswer( sql("SELECT substr(tableName, 1, 2) FROM tableName"), From 10b6c8cfa7d622d7728f61f53686215404e0ab59 Mon Sep 17 00:00:00 2001 From: xinyunh Date: Tue, 2 Sep 2014 10:21:21 -0700 Subject: [PATCH 5/7] fix the code style issue --- .../apache/spark/sql/catalyst/expressions/aggregates.scala | 3 ++- .../apache/spark/sql/catalyst/expressions/arithmetic.scala | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala index f519d96963e1c..1b4d892625dbb 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala @@ -514,5 +514,6 @@ case class LastFunction(expr: Expression, base: AggregateExpression) extends Agg result = input } - override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) else null + override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) + else null } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 25d99eb4aee14..7138c6e0e5456 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -153,7 +153,9 @@ case class Power(base: Expression, exponent: Expression) extends Expression { type EvaluatedType = Any def dataType: DataType = { - if (!resolved) throw new UnresolvedException(this, s"Cannot resolve since $children are not resolved") + if (!resolved) { + throw new UnresolvedException(this, s"Cannot resolve since $children are not resolved") + } DoubleType } override def foldable = base.foldable && exponent.foldable From 7f37dc8668de8404d8d415db96aa68daf78d8225 Mon Sep 17 00:00:00 2001 From: xinyunh Date: Wed, 3 Sep 2014 11:05:11 -0700 Subject: [PATCH 6/7] only remain POWER part --- .../apache/spark/sql/catalyst/SqlParser.scala | 4 --- .../sql/catalyst/expressions/aggregates.scala | 28 ------------------- .../sql/catalyst/expressions/arithmetic.scala | 14 ---------- .../org/apache/spark/sql/SQLQuerySuite.scala | 19 ------------- 4 files changed, 65 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index c10445c0f691a..c478b3810d6a0 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -81,7 +81,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val DISTINCT = Keyword("DISTINCT") protected val FALSE = Keyword("FALSE") protected val FIRST = Keyword("FIRST") - protected val LAST = Keyword("LAST") protected val FROM = Keyword("FROM") protected val FULL = Keyword("FULL") protected val GROUP = Keyword("GROUP") @@ -125,7 +124,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val SUBSTR = Keyword("SUBSTR") protected val SUBSTRING = Keyword("SUBSTRING") protected val SQRT = Keyword("SQRT") - protected val ABS = Keyword("ABS") protected val POW = Keyword("POW") protected val POWER = Keyword("POWER") @@ -315,7 +313,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ _ ~ _ ~ _ ~ _ ~ e => ApproxCountDistinct(e, s.toDouble) } | FIRST ~> "(" ~> expression <~ ")" ^^ { case exp => First(exp) } | - LAST ~> "(" ~> expression <~ ")" ^^ { case exp => Last(exp) } | AVG ~> "(" ~> expression <~ ")" ^^ { case exp => Average(exp) } | MIN ~> "(" ~> expression <~ ")" ^^ { case exp => Min(exp) } | MAX ~> "(" ~> expression <~ ")" ^^ { case exp => Max(exp) } | @@ -331,7 +328,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ "," ~ p ~ "," ~ l => Substring(s,p,l) } | SQRT ~> "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) } | - ABS ~> "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) } | (POW | POWER) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ { case s ~ "," ~ p => Power(s,p) } | diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala index 1b4d892625dbb..15560a2a933ad 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala @@ -344,21 +344,6 @@ case class First(child: Expression) extends PartialAggregate with trees.UnaryNod override def newInstance() = new FirstFunction(child, this) } -case class Last(child: Expression) extends PartialAggregate with trees.UnaryNode[Expression] { - override def references = child.references - override def nullable = true - override def dataType = child.dataType - override def toString = s"LAST($child)" - - override def asPartial: SplitEvaluation = { - val partialLast = Alias(Last(child), "PartialLast")() - SplitEvaluation( - Last(partialLast.toAttribute), - partialLast :: Nil) - } - override def newInstance() = new LastFunction(child, this) -} - case class AverageFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { @@ -504,16 +489,3 @@ case class FirstFunction(expr: Expression, base: AggregateExpression) extends Ag override def eval(input: Row): Any = result } - -case class LastFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { - def this() = this(null, null) // Required for serialization. - - var result: Any = null - - override def update(input: Row): Unit = { - result = input - } - - override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) - else null -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 7138c6e0e5456..6c6c1b4dbb858 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -131,20 +131,6 @@ case class MaxOf(left: Expression, right: Expression) extends Expression { override def toString = s"MaxOf($left, $right)" } -/** - * A function that get the absolute value of the numeric value. - */ -case class Abs(child: Expression) extends UnaryExpression { - type EvaluatedType = Any - - def dataType = child.dataType - override def foldable = child.foldable - def nullable = child.nullable - override def toString = s"Abs($child)" - - override def eval(input: Row): Any = n1(child, input, _.abs(_)) -} - /** * A function that get the power value of two parameters. * First one is taken as base while second one taken as exponent diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 4a6842c66ebd7..7c49020df905f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -41,18 +41,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { } - test("SPARK-3176 Added Parser of SQL ABS()") { - checkAnswer( - sql("SELECT ABS(-1.3)"), - 1.3) - checkAnswer( - sql("SELECT ABS(0.0)"), - 0.0) - checkAnswer( - sql("SELECT ABS(2.5)"), - 2.5) - } - test("SPARK-3176 Added Parser of SQL POWER()") { checkAnswer( sql("SELECT POWER(0, 512.0)"), @@ -86,13 +74,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { 4.0) } - test("SPARK-3176 Added Parser of SQL LAST()") { - checkAnswer( - sql("SELECT LAST(n) FROM lowerCaseData"), - 4) - } - - test("SPARK-2041 column name equals tablename") { checkAnswer( sql("SELECT tableName FROM tableName"), From 0c038fc58ac7fa237e0a0eb5b93409c9474643a7 Mon Sep 17 00:00:00 2001 From: xinyunh Date: Wed, 3 Sep 2014 11:05:11 -0700 Subject: [PATCH 7/7] only remain POWER part --- .../apache/spark/sql/catalyst/SqlParser.scala | 4 --- .../spark/sql/catalyst/dsl/package.scala | 1 - .../sql/catalyst/expressions/aggregates.scala | 28 ------------------- .../sql/catalyst/expressions/arithmetic.scala | 14 ---------- .../org/apache/spark/sql/SQLQuerySuite.scala | 19 ------------- 5 files changed, 66 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala index c10445c0f691a..c478b3810d6a0 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala @@ -81,7 +81,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val DISTINCT = Keyword("DISTINCT") protected val FALSE = Keyword("FALSE") protected val FIRST = Keyword("FIRST") - protected val LAST = Keyword("LAST") protected val FROM = Keyword("FROM") protected val FULL = Keyword("FULL") protected val GROUP = Keyword("GROUP") @@ -125,7 +124,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { protected val SUBSTR = Keyword("SUBSTR") protected val SUBSTRING = Keyword("SUBSTRING") protected val SQRT = Keyword("SQRT") - protected val ABS = Keyword("ABS") protected val POW = Keyword("POW") protected val POWER = Keyword("POWER") @@ -315,7 +313,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ _ ~ _ ~ _ ~ _ ~ e => ApproxCountDistinct(e, s.toDouble) } | FIRST ~> "(" ~> expression <~ ")" ^^ { case exp => First(exp) } | - LAST ~> "(" ~> expression <~ ")" ^^ { case exp => Last(exp) } | AVG ~> "(" ~> expression <~ ")" ^^ { case exp => Average(exp) } | MIN ~> "(" ~> expression <~ ")" ^^ { case exp => Min(exp) } | MAX ~> "(" ~> expression <~ ")" ^^ { case exp => Max(exp) } | @@ -331,7 +328,6 @@ class SqlParser extends StandardTokenParsers with PackratParsers { case s ~ "," ~ p ~ "," ~ l => Substring(s,p,l) } | SQRT ~> "(" ~> expression <~ ")" ^^ { case exp => Sqrt(exp) } | - ABS ~> "(" ~> expression <~ ")" ^^ { case exp => Abs(exp) } | (POW | POWER) ~> "(" ~> expression ~ "," ~ expression <~ ")" ^^ { case s ~ "," ~ p => Power(s,p) } | diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala index deb622c39faf5..f44521d6381c9 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/dsl/package.scala @@ -132,7 +132,6 @@ package object dsl { def approxCountDistinct(e: Expression, rsd: Double = 0.05) = ApproxCountDistinct(e, rsd) def avg(e: Expression) = Average(e) def first(e: Expression) = First(e) - def last(e: Expression) = Last(e) def min(e: Expression) = Min(e) def max(e: Expression) = Max(e) def upper(e: Expression) = Upper(e) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala index 1b4d892625dbb..15560a2a933ad 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala @@ -344,21 +344,6 @@ case class First(child: Expression) extends PartialAggregate with trees.UnaryNod override def newInstance() = new FirstFunction(child, this) } -case class Last(child: Expression) extends PartialAggregate with trees.UnaryNode[Expression] { - override def references = child.references - override def nullable = true - override def dataType = child.dataType - override def toString = s"LAST($child)" - - override def asPartial: SplitEvaluation = { - val partialLast = Alias(Last(child), "PartialLast")() - SplitEvaluation( - Last(partialLast.toAttribute), - partialLast :: Nil) - } - override def newInstance() = new LastFunction(child, this) -} - case class AverageFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { @@ -504,16 +489,3 @@ case class FirstFunction(expr: Expression, base: AggregateExpression) extends Ag override def eval(input: Row): Any = result } - -case class LastFunction(expr: Expression, base: AggregateExpression) extends AggregateFunction { - def this() = this(null, null) // Required for serialization. - - var result: Any = null - - override def update(input: Row): Unit = { - result = input - } - - override def eval(input: Row): Any = if (result != null) expr.eval(result.asInstanceOf[Row]) - else null -} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 7138c6e0e5456..6c6c1b4dbb858 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -131,20 +131,6 @@ case class MaxOf(left: Expression, right: Expression) extends Expression { override def toString = s"MaxOf($left, $right)" } -/** - * A function that get the absolute value of the numeric value. - */ -case class Abs(child: Expression) extends UnaryExpression { - type EvaluatedType = Any - - def dataType = child.dataType - override def foldable = child.foldable - def nullable = child.nullable - override def toString = s"Abs($child)" - - override def eval(input: Row): Any = n1(child, input, _.abs(_)) -} - /** * A function that get the power value of two parameters. * First one is taken as base while second one taken as exponent diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 4a6842c66ebd7..7c49020df905f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -41,18 +41,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { } - test("SPARK-3176 Added Parser of SQL ABS()") { - checkAnswer( - sql("SELECT ABS(-1.3)"), - 1.3) - checkAnswer( - sql("SELECT ABS(0.0)"), - 0.0) - checkAnswer( - sql("SELECT ABS(2.5)"), - 2.5) - } - test("SPARK-3176 Added Parser of SQL POWER()") { checkAnswer( sql("SELECT POWER(0, 512.0)"), @@ -86,13 +74,6 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll { 4.0) } - test("SPARK-3176 Added Parser of SQL LAST()") { - checkAnswer( - sql("SELECT LAST(n) FROM lowerCaseData"), - 4) - } - - test("SPARK-2041 column name equals tablename") { checkAnswer( sql("SELECT tableName FROM tableName"),