Skip to content

Commit e29b08a

Browse files
hvanhovellcloud-fan
authored andcommitted
[SPARK-23208][SQL] Fix code generation for complex create array (related) expressions
## What changes were proposed in this pull request? The `GenArrayData.genCodeToCreateArrayData` produces illegal java code when code splitting is enabled. This is used in `CreateArray` and `CreateMap` expressions for complex object arrays. This issue is caused by a typo. ## How was this patch tested? Added a regression test in `complexTypesSuite`. Author: Herman van Hovell <[email protected]> Closes #20391 from hvanhovell/SPARK-23208.
1 parent 45b4bbf commit e29b08a

File tree

2 files changed

+8
-2
lines changed

2 files changed

+8
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ private [sql] object GenArrayData {
111111
val assignmentString = ctx.splitExpressionsWithCurrentInputs(
112112
expressions = assignments,
113113
funcName = "apply",
114-
extraArguments = ("Object[]", arrayDataName) :: Nil)
114+
extraArguments = ("Object[]", arrayName) :: Nil)
115115

116116
(s"Object[] $arrayName = new Object[$numElements];",
117117
assignmentString,

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,15 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
2424
import org.apache.spark.sql.catalyst.plans.PlanTest
2525
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range}
2626
import org.apache.spark.sql.catalyst.rules.RuleExecutor
27+
import org.apache.spark.sql.catalyst.util.GenericArrayData
2728
import org.apache.spark.sql.types._
2829

2930
/**
3031
* SPARK-18601 discusses simplification direct access to complex types creators.
3132
* i.e. {{{create_named_struct(square, `x` * `x`).square}}} can be simplified to {{{`x` * `x`}}}.
3233
* sam applies to create_array and create_map
3334
*/
34-
class ComplexTypesSuite extends PlanTest{
35+
class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
3536

3637
object Optimizer extends RuleExecutor[LogicalPlan] {
3738
val batches =
@@ -171,6 +172,11 @@ class ComplexTypesSuite extends PlanTest{
171172
assert(ctx.inlinedMutableStates.length == 0)
172173
}
173174

175+
test("SPARK-23208: Test code splitting for create array related methods") {
176+
val inputs = (1 to 2500).map(x => Literal(s"l_$x"))
177+
checkEvaluation(CreateArray(inputs), new GenericArrayData(inputs.map(_.eval())))
178+
}
179+
174180
test("simplify map ops") {
175181
val rel = relation
176182
.select(

0 commit comments

Comments
 (0)