Skip to content

Commit cbd4028

Browse files
rxinrxin
committed
[SPARK-7069][SQL] Rename NativeType -> AtomicType.
Also renamed JvmType to InternalType.
1 parent 29163c5 commit cbd4028

24 files changed

+159
-153
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717

1818
package org.apache.spark.sql.catalyst
1919

20-
import java.sql.Timestamp
21-
2220
import org.apache.spark.util.Utils
2321
import org.apache.spark.sql.catalyst.expressions._
2422
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
@@ -110,7 +108,7 @@ trait ScalaReflection {
110108
StructField(p.name.toString, dataType, nullable)
111109
}), nullable = true)
112110
case t if t <:< typeOf[String] => Schema(StringType, nullable = true)
113-
case t if t <:< typeOf[Timestamp] => Schema(TimestampType, nullable = true)
111+
case t if t <:< typeOf[java.sql.Timestamp] => Schema(TimestampType, nullable = true)
114112
case t if t <:< typeOf[java.sql.Date] => Schema(DateType, nullable = true)
115113
case t if t <:< typeOf[BigDecimal] => Schema(DecimalType.Unlimited, nullable = true)
116114
case t if t <:< typeOf[java.math.BigDecimal] => Schema(DecimalType.Unlimited, nullable = true)
@@ -136,20 +134,20 @@ trait ScalaReflection {
136134

137135
def typeOfObject: PartialFunction[Any, DataType] = {
138136
// The data type can be determined without ambiguity.
139-
case obj: BooleanType.JvmType => BooleanType
140-
case obj: BinaryType.JvmType => BinaryType
137+
case obj: Boolean => BooleanType
138+
case obj: Array[Byte] => BinaryType
141139
case obj: String => StringType
142-
case obj: StringType.JvmType => StringType
143-
case obj: ByteType.JvmType => ByteType
144-
case obj: ShortType.JvmType => ShortType
145-
case obj: IntegerType.JvmType => IntegerType
146-
case obj: LongType.JvmType => LongType
147-
case obj: FloatType.JvmType => FloatType
148-
case obj: DoubleType.JvmType => DoubleType
140+
case obj: UTF8String => StringType
141+
case obj: Byte => ByteType
142+
case obj: Short => ShortType
143+
case obj: Int => IntegerType
144+
case obj: Long => LongType
145+
case obj: Float => FloatType
146+
case obj: Double => DoubleType
149147
case obj: java.sql.Date => DateType
150148
case obj: java.math.BigDecimal => DecimalType.Unlimited
151149
case obj: Decimal => DecimalType.Unlimited
152-
case obj: TimestampType.JvmType => TimestampType
150+
case obj: java.sql.Timestamp => TimestampType
153151
case null => NullType
154152
// For other cases, there is no obvious mapping from the type of the given object to a
155153
// Catalyst data type. A user should provide his/her specific rules

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ case class MaxOf(left: Expression, right: Expression) extends Expression {
346346
}
347347

348348
lazy val ordering = left.dataType match {
349-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
349+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
350350
case other => sys.error(s"Type $other does not support ordered operations")
351351
}
352352

@@ -391,7 +391,7 @@ case class MinOf(left: Expression, right: Expression) extends Expression {
391391
}
392392

393393
lazy val ordering = left.dataType match {
394-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
394+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
395395
case other => sys.error(s"Type $other does not support ordered operations")
396396
}
397397

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -623,7 +623,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
623623
protected def getColumn(inputRow: TermName, dataType: DataType, ordinal: Int) = {
624624
dataType match {
625625
case StringType => q"$inputRow($ordinal).asInstanceOf[org.apache.spark.sql.types.UTF8String]"
626-
case dt @ NativeType() => q"$inputRow.${accessorForType(dt)}($ordinal)"
626+
case dt: DataType if isNativeType(dt) => q"$inputRow.${accessorForType(dt)}($ordinal)"
627627
case _ => q"$inputRow.apply($ordinal).asInstanceOf[${termForType(dataType)}]"
628628
}
629629
}
@@ -635,7 +635,8 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
635635
value: TermName) = {
636636
dataType match {
637637
case StringType => q"$destinationRow.update($ordinal, $value)"
638-
case dt @ NativeType() => q"$destinationRow.${mutatorForType(dt)}($ordinal, $value)"
638+
case dt: DataType if isNativeType(dt) =>
639+
q"$destinationRow.${mutatorForType(dt)}($ordinal, $value)"
639640
case _ => q"$destinationRow.update($ordinal, $value)"
640641
}
641642
}
@@ -675,7 +676,18 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
675676
}
676677

677678
protected def termForType(dt: DataType) = dt match {
678-
case n: NativeType => n.tag
679+
case n: AtomicType => n.tag
679680
case _ => typeTag[Any]
680681
}
682+
683+
/**
684+
* List of data types that have special accessors and setters in [[Row]].
685+
*/
686+
protected val nativeTypes =
687+
Seq(IntegerType, BooleanType, LongType, DoubleType, FloatType, ShortType, ByteType, StringType)
688+
689+
/**
690+
* Returns true if the data type has a special accessor and setter in [[Row]].
691+
*/
692+
protected def isNativeType(dt: DataType) = nativeTypes.contains(dt)
681693
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
109109
q"override def update(i: Int, value: Any): Unit = { ..$cases; $accessorFailure }"
110110
}
111111

112-
val specificAccessorFunctions = NativeType.all.map { dataType =>
112+
val specificAccessorFunctions = nativeTypes.map { dataType =>
113113
val ifStatements = expressions.zipWithIndex.flatMap {
114114
// getString() is not used by expressions
115115
case (e, i) if e.dataType == dataType && dataType != StringType =>
@@ -135,7 +135,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
135135
}
136136
}
137137

138-
val specificMutatorFunctions = NativeType.all.map { dataType =>
138+
val specificMutatorFunctions = nativeTypes.map { dataType =>
139139
val ifStatements = expressions.zipWithIndex.flatMap {
140140
// setString() is not used by expressions
141141
case (e, i) if e.dataType == dataType && dataType != StringType =>

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
2020
import org.apache.spark.sql.catalyst.analysis.UnresolvedException
2121
import org.apache.spark.sql.catalyst.errors.TreeNodeException
2222
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
23-
import org.apache.spark.sql.types.{DataType, BinaryType, BooleanType, NativeType}
23+
import org.apache.spark.sql.types.{DataType, BinaryType, BooleanType, AtomicType}
2424

2525
object InterpretedPredicate {
2626
def create(expression: Expression, inputSchema: Seq[Attribute]): (Row => Boolean) =
@@ -211,7 +211,7 @@ case class LessThan(left: Expression, right: Expression) extends BinaryCompariso
211211
s"Types do not match ${left.dataType} != ${right.dataType}")
212212
}
213213
left.dataType match {
214-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
214+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
215215
case other => sys.error(s"Type $other does not support ordered operations")
216216
}
217217
}
@@ -240,7 +240,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) extends BinaryCo
240240
s"Types do not match ${left.dataType} != ${right.dataType}")
241241
}
242242
left.dataType match {
243-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
243+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
244244
case other => sys.error(s"Type $other does not support ordered operations")
245245
}
246246
}
@@ -269,7 +269,7 @@ case class GreaterThan(left: Expression, right: Expression) extends BinaryCompar
269269
s"Types do not match ${left.dataType} != ${right.dataType}")
270270
}
271271
left.dataType match {
272-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
272+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
273273
case other => sys.error(s"Type $other does not support ordered operations")
274274
}
275275
}
@@ -298,7 +298,7 @@ case class GreaterThanOrEqual(left: Expression, right: Expression) extends Binar
298298
s"Types do not match ${left.dataType} != ${right.dataType}")
299299
}
300300
left.dataType match {
301-
case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
301+
case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
302302
case other => sys.error(s"Type $other does not support ordered operations")
303303
}
304304
}

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.sql.catalyst.expressions
1919

20-
import org.apache.spark.sql.types.{UTF8String, DataType, StructType, NativeType}
20+
import org.apache.spark.sql.types.{UTF8String, DataType, StructType, AtomicType}
2121

2222
/**
2323
* An extended interface to [[Row]] that allows the values for each column to be updated. Setting
@@ -227,9 +227,9 @@ class RowOrdering(ordering: Seq[SortOrder]) extends Ordering[Row] {
227227
return if (order.direction == Ascending) 1 else -1
228228
} else {
229229
val comparison = order.dataType match {
230-
case n: NativeType if order.direction == Ascending =>
230+
case n: AtomicType if order.direction == Ascending =>
231231
n.ordering.asInstanceOf[Ordering[Any]].compare(left, right)
232-
case n: NativeType if order.direction == Descending =>
232+
case n: AtomicType if order.direction == Descending =>
233233
n.ordering.asInstanceOf[Ordering[Any]].reverse.compare(left, right)
234234
case other => sys.error(s"Type $other does not support ordered operations")
235235
}

0 commit comments

Comments
 (0)