Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ class SqlLexical extends StdLexical {
( identChar ~ (identChar | digit).* ^^
{ case first ~ rest => processIdent((first :: rest).mkString) }
| rep1(digit) ~ ('.' ~> digit.*).? ^^ {
case i ~ None => NumericLit(i.mkString)
case i ~ None => NumericLit(i.mkString)
case i ~ Some(d) => FloatLit(i.mkString + "." + d.mkString)
}
| '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
(HAVING ~> expression).? ~
sortType.? ~
(LIMIT ~> expression).? ^^ {
case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
case d ~ p ~ r ~ f ~ g ~ h ~ o ~ l =>
val base = r.getOrElse(OneRowRelation)
val withFilter = f.map(Filter(_, base)).getOrElse(base)
val withProjection = g
Expand Down Expand Up @@ -212,7 +212,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {

protected lazy val ordering: Parser[Seq[SortOrder]] =
( rep1sep(expression ~ direction.? , ",") ^^ {
case exps => exps.map(pair => SortOrder(pair._1, pair._2.getOrElse(Ascending)))
case exps => exps.map(pair => SortOrder(pair._1, pair._2.getOrElse(Ascending)))
}
)

Expand Down Expand Up @@ -242,7 +242,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
| termExpression ~ NOT.? ~ (BETWEEN ~> termExpression) ~ (AND ~> termExpression) ^^ {
case e ~ not ~ el ~ eu =>
val betweenExpr: Expression = And(GreaterThanOrEqual(e, el), LessThanOrEqual(e, eu))
not.fold(betweenExpr)(f=> Not(betweenExpr))
not.fold(betweenExpr)(f => Not(betweenExpr))
}
| termExpression ~ (RLIKE ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
| termExpression ~ (REGEXP ~> termExpression) ^^ { case e1 ~ e2 => RLike(e1, e2) }
Expand Down Expand Up @@ -365,7 +365,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {

protected lazy val baseExpression: Parser[Expression] =
( "*" ^^^ UnresolvedStar(None)
| ident <~ "." ~ "*" ^^ { case tableName => UnresolvedStar(Option(tableName)) }
| ident <~ "." ~ "*" ^^ { case tableName => UnresolvedStar(Option(tableName)) }
| primary
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ class Analyzer(
def apply(plan: LogicalPlan): LogicalPlan = plan transformUp {
case filter @ Filter(havingCondition, aggregate @ Aggregate(_, originalAggExprs, _))
if aggregate.resolved && containsAggregate(havingCondition) => {
val evaluatedCondition = Alias(havingCondition, "havingCondition")()
val evaluatedCondition = Alias(havingCondition, "havingCondition")()
val aggExprsWithHaving = evaluatedCondition +: originalAggExprs

Project(aggregate.output,
Expand All @@ -515,16 +515,15 @@ class Analyzer(
* - concrete attribute references for their output.
* - to be relocated from a SELECT clause (i.e. from a [[Project]]) into a [[Generate]]).
*
* Names for the output [[Attributes]] are extracted from [[Alias]] or [[MultiAlias]] expressions
* Names for the output [[Attribute]]s are extracted from [[Alias]] or [[MultiAlias]] expressions
* that wrap the [[Generator]]. If more than one [[Generator]] is found in a Project, an
* [[AnalysisException]] is throw.
*/
object ResolveGenerate extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
case p: Generate if !p.child.resolved || !p.generator.resolved => p
case g: Generate if g.resolved == false =>
g.copy(
generatorOutput = makeGeneratorOutput(g.generator, g.generatorOutput.map(_.name)))
case g: Generate if !g.resolved =>
g.copy(generatorOutput = makeGeneratorOutput(g.generator, g.generatorOutput.map(_.name)))

case p @ Project(projectList, child) =>
// Holds the resolved generator, if one exists in the project list.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ class SimpleCatalog(val conf: CatalystConf) extends Catalog {
trait OverrideCatalog extends Catalog {

// TODO: This doesn't work when the database changes...
val overrides = new mutable.HashMap[(Option[String],String), LogicalPlan]()
val overrides = new mutable.HashMap[(Option[String], String), LogicalPlan]()

abstract override def tableExists(tableIdentifier: Seq[String]): Boolean = {
val tableIdent = processTableIdentifier(tableIdentifier)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -561,8 +561,7 @@ trait HiveTypeCoercion {

case a @ CreateArray(children) if !a.resolved =>
val commonType = a.childTypes.reduce(
(a,b) =>
findTightestCommonType(a,b).getOrElse(StringType))
(a, b) => findTightestCommonType(a, b).getOrElse(StringType))
CreateArray(
children.map(c => if (c.dataType == commonType) c else Cast(c, commonType)))

Expand Down Expand Up @@ -634,7 +633,7 @@ trait HiveTypeCoercion {
import HiveTypeCoercion._

def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions {
case cw: CaseWhenLike if !cw.resolved && cw.childrenResolved && !cw.valueTypesEqual =>
case cw: CaseWhenLike if !cw.resolved && cw.childrenResolved && !cw.valueTypesEqual =>
logDebug(s"Input values for null casting ${cw.valueTypes.mkString(",")}")
val commonType = cw.valueTypes.reduce { (v1, v2) =>
findTightestCommonType(v1, v2).getOrElse(sys.error(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ package object dsl {
// Note that if we make ExpressionConversions an object rather than a trait, we can
// then make this a value class to avoid the small penalty of runtime instantiation.
def $(args: Any*): analysis.UnresolvedAttribute = {
analysis.UnresolvedAttribute(sc.s(args :_*))
analysis.UnresolvedAttribute(sc.s(args : _*))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,4 @@ package object errors {
case e: Exception => throw new TreeNodeException(tree, msg, e)
}
}

/**
* Executes `f` which is expected to throw a
* [[catalyst.errors.TreeNodeException TreeNodeException]]. The first tree encountered in
* the stack of exceptions of type `TreeType` is returned.
*/
def getTree[TreeType <: TreeNode[_]](f: => Unit): TreeType = ??? // TODO: Implement
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,48 +35,48 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w

private[this] def forceNullable(from: DataType, to: DataType) = (from, to) match {
case (StringType, _: NumericType) => true
case (StringType, TimestampType) => true
case (DoubleType, TimestampType) => true
case (FloatType, TimestampType) => true
case (StringType, DateType) => true
case (_: NumericType, DateType) => true
case (BooleanType, DateType) => true
case (DateType, _: NumericType) => true
case (DateType, BooleanType) => true
case (StringType, TimestampType) => true
case (DoubleType, TimestampType) => true
case (FloatType, TimestampType) => true
case (StringType, DateType) => true
case (_: NumericType, DateType) => true
case (BooleanType, DateType) => true
case (DateType, _: NumericType) => true
case (DateType, BooleanType) => true
case (DoubleType, _: DecimalType) => true
case (FloatType, _: DecimalType) => true
case (FloatType, _: DecimalType) => true
case (_, DecimalType.Fixed(_, _)) => true // TODO: not all upcasts here can really give null
case _ => false
case _ => false
}

private[this] def resolvableNullability(from: Boolean, to: Boolean) = !from || to

private[this] def resolve(from: DataType, to: DataType): Boolean = {
(from, to) match {
case (from, to) if from == to => true
case (from, to) if from == to => true

case (NullType, _) => true
case (NullType, _) => true

case (_, StringType) => true
case (_, StringType) => true

case (StringType, BinaryType) => true
case (StringType, BinaryType) => true

case (StringType, BooleanType) => true
case (DateType, BooleanType) => true
case (TimestampType, BooleanType) => true
case (_: NumericType, BooleanType) => true
case (StringType, BooleanType) => true
case (DateType, BooleanType) => true
case (TimestampType, BooleanType) => true
case (_: NumericType, BooleanType) => true

case (StringType, TimestampType) => true
case (BooleanType, TimestampType) => true
case (DateType, TimestampType) => true
case (_: NumericType, TimestampType) => true
case (StringType, TimestampType) => true
case (BooleanType, TimestampType) => true
case (DateType, TimestampType) => true
case (_: NumericType, TimestampType) => true

case (_, DateType) => true
case (_, DateType) => true

case (StringType, _: NumericType) => true
case (BooleanType, _: NumericType) => true
case (DateType, _: NumericType) => true
case (TimestampType, _: NumericType) => true
case (StringType, _: NumericType) => true
case (BooleanType, _: NumericType) => true
case (DateType, _: NumericType) => true
case (TimestampType, _: NumericType) => true
case (_: NumericType, _: NumericType) => true

case (ArrayType(from, fn), ArrayType(to, tn)) =>
Expand Down Expand Up @@ -410,21 +410,21 @@ case class Cast(child: Expression, dataType: DataType) extends UnaryExpression w

private[this] def cast(from: DataType, to: DataType): Any => Any = to match {
case dt if dt == child.dataType => identity[Any]
case StringType => castToString(from)
case BinaryType => castToBinary(from)
case DateType => castToDate(from)
case decimal: DecimalType => castToDecimal(from, decimal)
case TimestampType => castToTimestamp(from)
case BooleanType => castToBoolean(from)
case ByteType => castToByte(from)
case ShortType => castToShort(from)
case IntegerType => castToInt(from)
case FloatType => castToFloat(from)
case LongType => castToLong(from)
case DoubleType => castToDouble(from)
case array: ArrayType => castArray(from.asInstanceOf[ArrayType], array)
case map: MapType => castMap(from.asInstanceOf[MapType], map)
case struct: StructType => castStruct(from.asInstanceOf[StructType], struct)
case StringType => castToString(from)
case BinaryType => castToBinary(from)
case DateType => castToDate(from)
case decimal: DecimalType => castToDecimal(from, decimal)
case TimestampType => castToTimestamp(from)
case BooleanType => castToBoolean(from)
case ByteType => castToByte(from)
case ShortType => castToShort(from)
case IntegerType => castToInt(from)
case FloatType => castToFloat(from)
case LongType => castToLong(from)
case DoubleType => castToDouble(from)
case array: ArrayType => castArray(from.asInstanceOf[ArrayType], array)
case map: MapType => castMap(from.asInstanceOf[MapType], map)
case struct: StructType => castStruct(from.asInstanceOf[StructType], struct)
}

private[this] lazy val cast: Any => Any = cast(child.dataType, dataType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ object ExtractValue {
case (ArrayType(StructType(fields), containsNull), Literal(fieldName, StringType)) =>
val ordinal = findField(fields, fieldName.toString, resolver)
GetArrayStructFields(child, fields(ordinal), ordinal, containsNull)
case (_: ArrayType, _) if extraction.dataType.isInstanceOf[IntegralType] =>
case (_: ArrayType, _) if extraction.dataType.isInstanceOf[IntegralType] =>
GetArrayItem(child, extraction)
case (_: MapType, _) =>
GetMapValue(child, extraction)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ case class MinFunction(expr: Expression, base: AggregateExpression) extends Aggr
override def update(input: Row): Unit = {
if (currentMin.value == null) {
currentMin.value = expr.eval(input)
} else if(cmp.eval(input) == true) {
} else if (cmp.eval(input) == true) {
currentMin.value = expr.eval(input)
}
}
Expand Down Expand Up @@ -142,7 +142,7 @@ case class MaxFunction(expr: Expression, base: AggregateExpression) extends Aggr
override def update(input: Row): Unit = {
if (currentMax.value == null) {
currentMax.value = expr.eval(input)
} else if(cmp.eval(input) == true) {
} else if (cmp.eval(input) == true) {
currentMax.value = expr.eval(input)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
// Uh, bad function name...
child.castOrNull(c => q"!$c", BooleanType)

case Add(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 + $eval2" }
case Add(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 + $eval2" }
case Subtract(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 - $eval2" }
case Multiply(e1, e2) => (e1, e2) evaluate { case (eval1, eval2) => q"$eval1 * $eval2" }
case Divide(e1, e2) =>
Expand Down Expand Up @@ -665,7 +665,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
protected def defaultPrimitive(dt: DataType) = dt match {
case BooleanType => ru.Literal(Constant(false))
case FloatType => ru.Literal(Constant(-1.0.toFloat))
case StringType => q"""org.apache.spark.sql.types.UTF8String("<uninit>")"""
case StringType => q"""org.apache.spark.sql.types.UTF8String("<uninit>")"""
case ShortType => ru.Literal(Constant(-1.toShort))
case LongType => ru.Literal(Constant(-1L))
case ByteType => ru.Literal(Constant(-1.toByte))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
}
}

val hashValues = expressions.zipWithIndex.map { case (e,i) =>
val hashValues = expressions.zipWithIndex.map { case (e, i) =>
val elementName = newTermName(s"c$i")
val nonNull = e.dataType match {
case BooleanType => q"if ($elementName) 0 else 1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ case class Explode(child: Expression)
val inputArray = child.eval(input).asInstanceOf[Seq[Any]]
if (inputArray == null) Nil else inputArray.map(v => new GenericRow(Array(v)))
case MapType(_, _, _) =>
val inputMap = child.eval(input).asInstanceOf[Map[Any,Any]]
if (inputMap == null) Nil else inputMap.map { case (k,v) => new GenericRow(Array(k,v)) }
val inputMap = child.eval(input).asInstanceOf[Map[Any, Any]]
if (inputMap == null) Nil else inputMap.map { case (k, v) => new GenericRow(Array(k, v)) }
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ trait StringRegexExpression extends ExpectsInputTypes {
case _ => null
}

protected def compile(str: String): Pattern = if(str == null) {
protected def compile(str: String): Pattern = if (str == null) {
null
} else {
// Let it raise exception if couldn't compile the regex string
Pattern.compile(escape(str))
}

protected def pattern(str: String) = if(cache == null) compile(str) else cache
protected def pattern(str: String) = if (cache == null) compile(str) else cache

override def eval(input: Row): Any = {
val l = left.eval(input)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ case class WindowSpecDefinition(
}
}

override def children: Seq[Expression] = partitionSpec ++ orderSpec
override def children: Seq[Expression] = partitionSpec ++ orderSpec

override lazy val resolved: Boolean =
childrenResolved && frameSpecification.isInstanceOf[SpecifiedWindowFrame]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ abstract class QueryPlan[PlanType <: TreeNode[PlanType]] extends TreeNode[PlanTy
val newArgs = productIterator.map {
case e: Expression => transformExpressionDown(e)
case Some(e: Expression) => Some(transformExpressionDown(e))
case m: Map[_,_] => m
case m: Map[_, _] => m
case d: DataType => d // Avoid unpacking Structs
case seq: Traversable[_] => seq.map {
case e: Expression => transformExpressionDown(e)
Expand Down Expand Up @@ -117,7 +117,7 @@ abstract class QueryPlan[PlanType <: TreeNode[PlanType]] extends TreeNode[PlanTy
val newArgs = productIterator.map {
case e: Expression => transformExpressionUp(e)
case Some(e: Expression) => Some(transformExpressionUp(e))
case m: Map[_,_] => m
case m: Map[_, _] => m
case d: DataType => d // Avoid unpacking Structs
case seq: Traversable[_] => seq.map {
case e: Expression => transformExpressionUp(e)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ case class Union(left: LogicalPlan, right: LogicalPlan) extends BinaryNode {

override lazy val resolved: Boolean =
childrenResolved &&
left.output.zip(right.output).forall { case (l,r) => l.dataType == r.dataType }
left.output.zip(right.output).forall { case (l, r) => l.dataType == r.dataType }

override def statistics: Statistics = {
val sizeInBytes = left.statistics.sizeInBytes + right.statistics.sizeInBytes
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] {
} else {
Some(arg)
}
case m: Map[_,_] => m
case m: Map[_, _] => m
case d: DataType => d // Avoid unpacking Structs
case args: Traversable[_] => args.map {
case arg: TreeNode[_] if children contains arg =>
Expand Down Expand Up @@ -311,7 +311,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] {
} else {
Some(arg)
}
case m: Map[_,_] => m
case m: Map[_, _] => m
case d: DataType => d // Avoid unpacking Structs
case args: Traversable[_] => args.map {
case arg: TreeNode[_] if children contains arg =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ package object util {
}

def resourceToString(
resource:String,
resource: String,
encoding: String = "UTF-8",
classLoader: ClassLoader = Utils.getSparkClassLoader): String = {
new String(resourceToBytes(resource, classLoader), encoding)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ object DataType {

protected lazy val structField: Parser[StructField] =
("StructField(" ~> "[a-zA-Z0-9_]*".r) ~ ("," ~> dataType) ~ ("," ~> boolVal <~ ")") ^^ {
case name ~ tpe ~ nullable =>
case name ~ tpe ~ nullable =>
StructField(name, tpe, nullable = nullable)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ class ScalaReflectionSuite extends FunSuite {
}

assert(ArrayType(IntegerType) === typeOfObject3(Seq(1, 2, 3)))
assert(ArrayType(ArrayType(IntegerType)) === typeOfObject3(Seq(Seq(1,2,3))))
assert(ArrayType(ArrayType(IntegerType)) === typeOfObject3(Seq(Seq(1, 2, 3))))
}

test("convert PrimitiveData to catalyst") {
Expand Down
Loading