Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.types.Decimal;
import org.apache.spark.unsafe.types.Interval;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;

public interface SpecializedGetters {
Expand All @@ -46,7 +46,7 @@ public interface SpecializedGetters {

byte[] getBinary(int ordinal);

Interval getInterval(int ordinal);
CalendarInterval getInterval(int ordinal);

InternalRow getStruct(int ordinal, int numFields);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.bitset.BitSetMethods;
import org.apache.spark.unsafe.hash.Murmur3_x86_32;
import org.apache.spark.unsafe.types.Interval;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;

import static org.apache.spark.sql.types.DataTypes.*;
Expand Down Expand Up @@ -92,7 +92,7 @@ public static int calculateBitSetWidthInBytes(int numFields) {
Arrays.asList(new DataType[]{
StringType,
BinaryType,
IntervalType
CalendarIntervalType
}));
_readableFieldTypes.addAll(settableFieldTypes);
readableFieldTypes = Collections.unmodifiableSet(_readableFieldTypes);
Expand Down Expand Up @@ -265,7 +265,7 @@ public Object get(int ordinal, DataType dataType) {
return getBinary(ordinal);
} else if (dataType instanceof StringType) {
return getUTF8String(ordinal);
} else if (dataType instanceof IntervalType) {
} else if (dataType instanceof CalendarIntervalType) {
return getInterval(ordinal);
} else if (dataType instanceof StructType) {
return getStruct(ordinal, ((StructType) dataType).size());
Expand Down Expand Up @@ -350,7 +350,7 @@ public byte[] getBinary(int ordinal) {
}

@Override
public Interval getInterval(int ordinal) {
public CalendarInterval getInterval(int ordinal) {
if (isNullAt(ordinal)) {
return null;
} else {
Expand All @@ -359,7 +359,7 @@ public Interval getInterval(int ordinal) {
final int months = (int) PlatformDependent.UNSAFE.getLong(baseObject, baseOffset + offset);
final long microseconds =
PlatformDependent.UNSAFE.getLong(baseObject, baseOffset + offset + 8);
return new Interval(months, microseconds);
return new CalendarInterval(months, microseconds);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.types.ByteArray;
import org.apache.spark.unsafe.types.Interval;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;

/**
Expand Down Expand Up @@ -131,7 +131,7 @@ public static int write(UnsafeRow target, int ordinal, int cursor, InternalRow i
/** Writer for interval type. */
public static class IntervalWriter {

public static int write(UnsafeRow target, int ordinal, int cursor, Interval input) {
public static int write(UnsafeRow target, int ordinal, int cursor, CalendarInterval input) {
final long offset = target.getBaseOffset() + cursor;

// Write the months and microseconds fields of Interval to the variable length portion.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@ public class DataTypes {
public static final DataType TimestampType = TimestampType$.MODULE$;

/**
* Gets the IntervalType object.
* Gets the CalendarIntervalType object.
*/
public static final DataType IntervalType = IntervalType$.MODULE$;
public static final DataType CalendarIntervalType = CalendarIntervalType$.MODULE$;

/**
* Gets the DoubleType object.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.{Interval, UTF8String}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}

/**
* An abstract class for row used internal in Spark SQL, which only contain the columns as
Expand Down Expand Up @@ -61,7 +61,8 @@ abstract class InternalRow extends Serializable with SpecializedGetters {
override def getDecimal(ordinal: Int): Decimal =
getAs[Decimal](ordinal, DecimalType.SYSTEM_DEFAULT)

override def getInterval(ordinal: Int): Interval = getAs[Interval](ordinal, IntervalType)
override def getInterval(ordinal: Int): CalendarInterval =
getAs[CalendarInterval](ordinal, CalendarIntervalType)

// This is only use for test and will throw a null pointer exception if the position is null.
def getString(ordinal: Int): String = getUTF8String(ordinal).toString
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.Interval
import org.apache.spark.unsafe.types.CalendarInterval

/**
* A very simple SQL parser. Based loosely on:
Expand Down Expand Up @@ -365,32 +365,32 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {

protected lazy val millisecond: Parser[Long] =
integral <~ intervalUnit("millisecond") ^^ {
case num => num.toLong * Interval.MICROS_PER_MILLI
case num => num.toLong * CalendarInterval.MICROS_PER_MILLI
}

protected lazy val second: Parser[Long] =
integral <~ intervalUnit("second") ^^ {
case num => num.toLong * Interval.MICROS_PER_SECOND
case num => num.toLong * CalendarInterval.MICROS_PER_SECOND
}

protected lazy val minute: Parser[Long] =
integral <~ intervalUnit("minute") ^^ {
case num => num.toLong * Interval.MICROS_PER_MINUTE
case num => num.toLong * CalendarInterval.MICROS_PER_MINUTE
}

protected lazy val hour: Parser[Long] =
integral <~ intervalUnit("hour") ^^ {
case num => num.toLong * Interval.MICROS_PER_HOUR
case num => num.toLong * CalendarInterval.MICROS_PER_HOUR
}

protected lazy val day: Parser[Long] =
integral <~ intervalUnit("day") ^^ {
case num => num.toLong * Interval.MICROS_PER_DAY
case num => num.toLong * CalendarInterval.MICROS_PER_DAY
}

protected lazy val week: Parser[Long] =
integral <~ intervalUnit("week") ^^ {
case num => num.toLong * Interval.MICROS_PER_WEEK
case num => num.toLong * CalendarInterval.MICROS_PER_WEEK
}

protected lazy val intervalLiteral: Parser[Literal] =
Expand All @@ -406,7 +406,7 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
val months = Seq(year, month).map(_.getOrElse(0)).sum
val microseconds = Seq(week, day, hour, minute, second, millisecond, microsecond)
.map(_.getOrElse(0L)).sum
Literal.create(new Interval(months, microseconds), IntervalType)
Literal.create(new CalendarInterval(months, microseconds), CalendarIntervalType)
}

private def toNarrowestIntegerType(value: String): Any = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ case class BoundReference(ordinal: Int, dataType: DataType, nullable: Boolean)
case DoubleType => input.getDouble(ordinal)
case StringType => input.getUTF8String(ordinal)
case BinaryType => input.getBinary(ordinal)
case IntervalType => input.getInterval(ordinal)
case CalendarIntervalType => input.getInterval(ordinal)
case t: StructType => input.getStruct(ordinal, t.size)
case _ => input.get(ordinal, dataType)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.{Interval, UTF8String}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}

import scala.collection.mutable

Expand Down Expand Up @@ -55,7 +55,7 @@ object Cast {

case (_, DateType) => true

case (StringType, IntervalType) => true
case (StringType, CalendarIntervalType) => true

case (StringType, _: NumericType) => true
case (BooleanType, _: NumericType) => true
Expand Down Expand Up @@ -225,7 +225,7 @@ case class Cast(child: Expression, dataType: DataType)
// IntervalConverter
private[this] def castToInterval(from: DataType): Any => Any = from match {
case StringType =>
buildCast[UTF8String](_, s => Interval.fromString(s.toString))
buildCast[UTF8String](_, s => CalendarInterval.fromString(s.toString))
case _ => _ => null
}

Expand Down Expand Up @@ -398,7 +398,7 @@ case class Cast(child: Expression, dataType: DataType)
case DateType => castToDate(from)
case decimal: DecimalType => castToDecimal(from, decimal)
case TimestampType => castToTimestamp(from)
case IntervalType => castToInterval(from)
case CalendarIntervalType => castToInterval(from)
case BooleanType => castToBoolean(from)
case ByteType => castToByte(from)
case ShortType => castToShort(from)
Expand Down Expand Up @@ -438,7 +438,7 @@ case class Cast(child: Expression, dataType: DataType)
case DateType => castToDateCode(from, ctx)
case decimal: DecimalType => castToDecimalCode(from, decimal)
case TimestampType => castToTimestampCode(from, ctx)
case IntervalType => castToIntervalCode(from)
case CalendarIntervalType => castToIntervalCode(from)
case BooleanType => castToBooleanCode(from)
case ByteType => castToByteCode(from)
case ShortType => castToShortCode(from)
Expand Down Expand Up @@ -630,7 +630,7 @@ case class Cast(child: Expression, dataType: DataType)
private[this] def castToIntervalCode(from: DataType): CastFunction = from match {
case StringType =>
(c, evPrim, evNull) =>
s"$evPrim = Interval.fromString($c.toString());"
s"$evPrim = CalendarInterval.fromString($c.toString());"
}

private[this] def decimalToTimestampCode(d: String): String =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.util.TypeUtils
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.Interval
import org.apache.spark.unsafe.types.CalendarInterval


case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes {
Expand All @@ -37,12 +37,12 @@ case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInp
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = dataType match {
case dt: DecimalType => defineCodeGen(ctx, ev, c => s"$c.unary_$$minus()")
case dt: NumericType => defineCodeGen(ctx, ev, c => s"(${ctx.javaType(dt)})(-($c))")
case dt: IntervalType => defineCodeGen(ctx, ev, c => s"$c.negate()")
case dt: CalendarIntervalType => defineCodeGen(ctx, ev, c => s"$c.negate()")
}

protected override def nullSafeEval(input: Any): Any = {
if (dataType.isInstanceOf[IntervalType]) {
input.asInstanceOf[Interval].negate()
if (dataType.isInstanceOf[CalendarIntervalType]) {
input.asInstanceOf[CalendarInterval].negate()
} else {
numeric.negate(input)
}
Expand Down Expand Up @@ -121,8 +121,8 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
private lazy val numeric = TypeUtils.getNumeric(dataType)

protected override def nullSafeEval(input1: Any, input2: Any): Any = {
if (dataType.isInstanceOf[IntervalType]) {
input1.asInstanceOf[Interval].add(input2.asInstanceOf[Interval])
if (dataType.isInstanceOf[CalendarIntervalType]) {
input1.asInstanceOf[CalendarInterval].add(input2.asInstanceOf[CalendarInterval])
} else {
numeric.plus(input1, input2)
}
Expand All @@ -134,7 +134,7 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
case ByteType | ShortType =>
defineCodeGen(ctx, ev,
(eval1, eval2) => s"(${ctx.javaType(dataType)})($eval1 $symbol $eval2)")
case IntervalType =>
case CalendarIntervalType =>
defineCodeGen(ctx, ev, (eval1, eval2) => s"$eval1.add($eval2)")
case _ =>
defineCodeGen(ctx, ev, (eval1, eval2) => s"$eval1 $symbol $eval2")
Expand All @@ -150,8 +150,8 @@ case class Subtract(left: Expression, right: Expression) extends BinaryArithmeti
private lazy val numeric = TypeUtils.getNumeric(dataType)

protected override def nullSafeEval(input1: Any, input2: Any): Any = {
if (dataType.isInstanceOf[IntervalType]) {
input1.asInstanceOf[Interval].subtract(input2.asInstanceOf[Interval])
if (dataType.isInstanceOf[CalendarIntervalType]) {
input1.asInstanceOf[CalendarInterval].subtract(input2.asInstanceOf[CalendarInterval])
} else {
numeric.minus(input1, input2)
}
Expand All @@ -163,7 +163,7 @@ case class Subtract(left: Expression, right: Expression) extends BinaryArithmeti
case ByteType | ShortType =>
defineCodeGen(ctx, ev,
(eval1, eval2) => s"(${ctx.javaType(dataType)})($eval1 $symbol $eval2)")
case IntervalType =>
case CalendarIntervalType =>
defineCodeGen(ctx, ev, (eval1, eval2) => s"$eval1.subtract($eval2)")
case _ =>
defineCodeGen(ctx, ev, (eval1, eval2) => s"$eval1 $symbol $eval2")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ class CodeGenContext {
case _ if isPrimitiveType(jt) => s"$row.get${primitiveTypeName(jt)}($ordinal)"
case StringType => s"$row.getUTF8String($ordinal)"
case BinaryType => s"$row.getBinary($ordinal)"
case IntervalType => s"$row.getInterval($ordinal)"
case CalendarIntervalType => s"$row.getInterval($ordinal)"
case t: StructType => s"$row.getStruct($ordinal, ${t.size})"
case _ => s"($jt)$row.get($ordinal)"
}
Expand Down Expand Up @@ -150,7 +150,7 @@ class CodeGenContext {
case dt: DecimalType => "Decimal"
case BinaryType => "byte[]"
case StringType => "UTF8String"
case IntervalType => "Interval"
case CalendarIntervalType => "CalendarInterval"
case _: StructType => "InternalRow"
case _: ArrayType => s"scala.collection.Seq"
case _: MapType => s"scala.collection.Map"
Expand Down Expand Up @@ -293,7 +293,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
classOf[UnsafeRow].getName,
classOf[UTF8String].getName,
classOf[Decimal].getName,
classOf[Interval].getName
classOf[CalendarInterval].getName
))
evaluator.setExtendedClass(classOf[GeneratedClass])
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
/** Returns true iff we support this data type. */
def canSupport(dataType: DataType): Boolean = dataType match {
case t: AtomicType if !t.isInstanceOf[DecimalType] => true
case _: IntervalType => true
case _: CalendarIntervalType => true
case t: StructType => t.toSeq.forall(field => canSupport(field.dataType))
case NullType => true
case _ => false
Expand Down Expand Up @@ -75,7 +75,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
s" + (${exprs(i).isNull} ? 0 : $StringWriter.getSize(${exprs(i).primitive}))"
case BinaryType =>
s" + (${exprs(i).isNull} ? 0 : $BinaryWriter.getSize(${exprs(i).primitive}))"
case IntervalType =>
case CalendarIntervalType =>
s" + (${exprs(i).isNull} ? 0 : 16)"
case _: StructType =>
s" + (${exprs(i).isNull} ? 0 : $StructWriter.getSize(${exprs(i).primitive}))"
Expand All @@ -91,7 +91,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
s"$cursor += $StringWriter.write($ret, $i, $cursor, ${exprs(i).primitive})"
case BinaryType =>
s"$cursor += $BinaryWriter.write($ret, $i, $cursor, ${exprs(i).primitive})"
case IntervalType =>
case CalendarIntervalType =>
s"$cursor += $IntervalWriter.write($ret, $i, $cursor, ${exprs(i).primitive})"
case t: StructType =>
s"$cursor += $StructWriter.write($ret, $i, $cursor, ${exprs(i).primitive})"
Expand Down Expand Up @@ -173,7 +173,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
s" + (${ev.isNull} ? 0 : $StringWriter.getSize(${ev.primitive}))"
case BinaryType =>
s" + (${ev.isNull} ? 0 : $BinaryWriter.getSize(${ev.primitive}))"
case IntervalType =>
case CalendarIntervalType =>
s" + (${ev.isNull} ? 0 : 16)"
case _: StructType =>
s" + (${ev.isNull} ? 0 : $StructWriter.getSize(${ev.primitive}))"
Expand All @@ -189,7 +189,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
s"$cursor += $StringWriter.write($primitive, $i, $cursor, ${exprs(i).primitive})"
case BinaryType =>
s"$cursor += $BinaryWriter.write($primitive, $i, $cursor, ${exprs(i).primitive})"
case IntervalType =>
case CalendarIntervalType =>
s"$cursor += $IntervalWriter.write($primitive, $i, $cursor, ${exprs(i).primitive})"
case t: StructType =>
s"$cursor += $StructWriter.write($primitive, $i, $cursor, ${exprs(i).primitive})"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ object Literal {
case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t), TimestampType)
case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType)
case a: Array[Byte] => Literal(a, BinaryType)
case i: Interval => Literal(i, IntervalType)
case i: CalendarInterval => Literal(i, CalendarIntervalType)
case null => Literal(null, NullType)
case _ =>
throw new RuntimeException("Unsupported literal type " + v.getClass + " " + v)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ private[sql] object TypeCollection {
* Types that include numeric types and interval type. They are only used in unary_minus,
* unary_positive, add and subtract operations.
*/
val NumericAndInterval = TypeCollection(NumericType, IntervalType)
val NumericAndInterval = TypeCollection(NumericType, CalendarIntervalType)

def apply(types: AbstractDataType*): TypeCollection = new TypeCollection(types)

Expand Down
Loading