Skip to content

Commit 77ed72d

Browse files
author
Davies Liu
committed
address comments
1 parent 4518e17 commit 77ed72d

File tree

1 file changed

+15
-17
lines changed

1 file changed

+15
-17
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ package org.apache.spark.sql.catalyst.expressions.codegen
1919

2020
import org.apache.spark.sql.catalyst.expressions._
2121
import org.apache.spark.sql.types._
22-
import org.apache.spark.unsafe.PlatformDependent
2322

2423
/**
2524
* Generates a [[Projection]] that returns an [[UnsafeRow]].
@@ -41,8 +40,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
4140
private val ArrayWriter = classOf[UnsafeRowWriters.ArrayWriter].getName
4241
private val MapWriter = classOf[UnsafeRowWriters.MapWriter].getName
4342

44-
private val PlatformDependent = classOf[PlatformDependent].getName
45-
4643
/** Returns true iff we support this data type. */
4744
def canSupport(dataType: DataType): Boolean = dataType match {
4845
case NullType => true
@@ -147,11 +144,12 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
147144
int $numBytes = $cursor + (${genAdditionalSize(dt, ev)});
148145
if ($buffer.length < $numBytes) {
149146
// This will not happen frequently, because the buffer is re-used.
150-
byte[] $tmp = new byte[$numBytes * 3 / 2];
151-
System.arraycopy($buffer, 0, $tmp, 0, $buffer.length);
147+
byte[] $tmp = new byte[$numBytes * 2];
148+
PlatformDependent.copyMemory($buffer, PlatformDependent.BYTE_ARRAY_OFFSET,
149+
$tmp, PlatformDependent.BYTE_ARRAY_OFFSET, $buffer.length);
152150
$buffer = $tmp;
153151
}
154-
$output.pointTo($buffer, $PlatformDependent.BYTE_ARRAY_OFFSET,
152+
$output.pointTo($buffer, PlatformDependent.BYTE_ARRAY_OFFSET,
155153
${inputTypes.length}, $numBytes);
156154
"""
157155
} else {
@@ -185,7 +183,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
185183

186184
val code = s"""
187185
$cursor = $fixedSize;
188-
$output.pointTo($buffer, $PlatformDependent.BYTE_ARRAY_OFFSET, ${inputTypes.length}, $cursor);
186+
$output.pointTo($buffer, PlatformDependent.BYTE_ARRAY_OFFSET, ${inputTypes.length}, $cursor);
189187
${ctx.splitExpressions(row, convertedFields)}
190188
"""
191189
GeneratedExpressionCode(code, "false", output)
@@ -269,17 +267,17 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
269267
// Should we do word align?
270268
val elementSize = elementType.defaultSize
271269
s"""
272-
$PlatformDependent.UNSAFE.put${ctx.primitiveTypeName(elementType)}(
270+
PlatformDependent.UNSAFE.put${ctx.primitiveTypeName(elementType)}(
273271
$buffer,
274-
$PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
272+
PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
275273
${convertedElement.primitive});
276274
$cursor += $elementSize;
277275
"""
278276
case t: DecimalType if t.precision <= Decimal.MAX_LONG_DIGITS =>
279277
s"""
280-
$PlatformDependent.UNSAFE.putLong(
278+
PlatformDependent.UNSAFE.putLong(
281279
$buffer,
282-
$PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
280+
PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
283281
${convertedElement.primitive}.toUnscaledLong());
284282
$cursor += 8;
285283
"""
@@ -288,7 +286,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
288286
s"""
289287
$cursor += $writer.write(
290288
$buffer,
291-
$PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
289+
PlatformDependent.BYTE_ARRAY_OFFSET + $cursor,
292290
$elements[$index]);
293291
"""
294292
}
@@ -322,14 +320,14 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
322320
for (int $index = 0; $index < $numElements; $index++) {
323321
if ($checkNull) {
324322
// If element is null, write the negative value address into offset region.
325-
$PlatformDependent.UNSAFE.putInt(
323+
PlatformDependent.UNSAFE.putInt(
326324
$buffer,
327-
$PlatformDependent.BYTE_ARRAY_OFFSET + 4 * $index,
325+
PlatformDependent.BYTE_ARRAY_OFFSET + 4 * $index,
328326
-$cursor);
329327
} else {
330-
$PlatformDependent.UNSAFE.putInt(
328+
PlatformDependent.UNSAFE.putInt(
331329
$buffer,
332-
$PlatformDependent.BYTE_ARRAY_OFFSET + 4 * $index,
330+
PlatformDependent.BYTE_ARRAY_OFFSET + 4 * $index,
333331
$cursor);
334332

335333
$writeElement
@@ -338,7 +336,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
338336

339337
$output.pointTo(
340338
$buffer,
341-
$PlatformDependent.BYTE_ARRAY_OFFSET,
339+
PlatformDependent.BYTE_ARRAY_OFFSET,
342340
$numElements,
343341
$numBytes);
344342
}

0 commit comments

Comments
 (0)