From 2b65856b78d663564a876f601d2f3bdfc7f353a6 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 20 Dec 2017 13:51:39 +0000 Subject: [PATCH 1/4] initial commit --- .../spark/sql/catalyst/expressions/generators.scala | 12 ++++++++---- .../apache/spark/sql/execution/GenerateExec.scala | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 1cd73a92a8635..b828a50344401 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -214,11 +214,15 @@ case class Stack(children: Seq[Expression]) extends Generator { // Create the collection. val wrapperClass = classOf[mutable.WrappedArray[_]].getName - ctx.addMutableState( + val wrappedArray = ctx.addMutableState( s"$wrapperClass", - ev.value, - v => s"$v = $wrapperClass$$.MODULE$$.make($rowData);", useFreshName = false) - ev.copy(code = code, isNull = "false") + "stackWrappedArray", + v => s"$v = $wrapperClass$$.MODULE$$.make($rowData);") + ev.copy(code = + s""" + |$code + |$wrapperClass ${ev.value} = $wrappedArray; + """.stripMargin, isNull = "false") } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala index e1562befe14f9..2a7fe9a5e7325 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala @@ -122,7 +122,7 @@ case class GenerateExec( } } - override def supportCodegen: Boolean = false + override def supportCodegen: Boolean = generator.supportCodegen override def inputRDDs(): Seq[RDD[InternalRow]] = { child.asInstanceOf[CodegenSupport].inputRDDs() From 81306c436a533b83d21c30bed8d8504ff66f8c76 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 20 Dec 2017 14:34:14 +0000 Subject: [PATCH 2/4] eliminate mutable states --- .../spark/sql/catalyst/expressions/generators.scala | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index b828a50344401..0f4024e8ff7c7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -199,8 +199,7 @@ case class Stack(children: Seq[Expression]) extends Generator { override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { // Rows - we write these into an array. - val rowData = ctx.addMutableState("InternalRow[]", "rows", - v => s"$v = new InternalRow[$numRows];") + val rowData = ctx.freshName("rowsStack") val values = children.tail val dataTypes = values.take(numFields).map(_.dataType) val code = ctx.splitExpressionsWithCurrentInputs(Seq.tabulate(numRows) { row => @@ -214,14 +213,11 @@ case class Stack(children: Seq[Expression]) extends Generator { // Create the collection. val wrapperClass = classOf[mutable.WrappedArray[_]].getName - val wrappedArray = ctx.addMutableState( - s"$wrapperClass", - "stackWrappedArray", - v => s"$v = $wrapperClass$$.MODULE$$.make($rowData);") ev.copy(code = s""" + |InternalRow[] $rowData = new InternalRow[$numRows]; |$code - |$wrapperClass ${ev.value} = $wrappedArray; + |$wrapperClass ${ev.value} = $wrapperClass$$.MODULE$$.make($rowData); """.stripMargin, isNull = "false") } } From d31ccd7b8aae1f46e7e9e5bfbb7d390c0992678c Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 20 Dec 2017 15:49:08 +0000 Subject: [PATCH 3/4] revert change for codegen in Generator --- .../scala/org/apache/spark/sql/execution/GenerateExec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala index 2a7fe9a5e7325..e1562befe14f9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala @@ -122,7 +122,7 @@ case class GenerateExec( } } - override def supportCodegen: Boolean = generator.supportCodegen + override def supportCodegen: Boolean = false override def inputRDDs(): Seq[RDD[InternalRow]] = { child.asInstanceOf[CodegenSupport].inputRDDs() From f0163e7b68aa09fef5c1dc7f25e00170354a1ab2 Mon Sep 17 00:00:00 2001 From: Kazuaki Ishizaki Date: Wed, 20 Dec 2017 17:15:11 +0000 Subject: [PATCH 4/4] revert change for rowData --- .../apache/spark/sql/catalyst/expressions/generators.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 0f4024e8ff7c7..69af7a250a5ac 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -199,7 +199,8 @@ case class Stack(children: Seq[Expression]) extends Generator { override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { // Rows - we write these into an array. - val rowData = ctx.freshName("rowsStack") + val rowData = ctx.addMutableState("InternalRow[]", "rows", + v => s"$v = new InternalRow[$numRows];") val values = children.tail val dataTypes = values.take(numFields).map(_.dataType) val code = ctx.splitExpressionsWithCurrentInputs(Seq.tabulate(numRows) { row => @@ -215,7 +216,6 @@ case class Stack(children: Seq[Expression]) extends Generator { val wrapperClass = classOf[mutable.WrappedArray[_]].getName ev.copy(code = s""" - |InternalRow[] $rowData = new InternalRow[$numRows]; |$code |$wrapperClass ${ev.value} = $wrapperClass$$.MODULE$$.make($rowData); """.stripMargin, isNull = "false")