From 40b70f3aea06e609366877e465b81a7e4463e0ee Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 28 May 2016 23:43:28 -0700 Subject: [PATCH 1/9] fix the boundary case. --- .../sql/catalyst/optimizer/Optimizer.scala | 3 +- .../spark/sql/internal/SQLConfSuite.scala | 54 ++++++++++++++++++- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 48d70099b6d8..4b31d7a10cbc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -937,7 +937,8 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { */ case class OptimizeCodegen(conf: CatalystConf) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case e @ CaseWhen(branches, _) if branches.size < conf.maxCaseBranchesForCodegen => + case e @ CaseWhen(branches, elseBranch) + if branches.size + elseBranch.size <= conf.maxCaseBranchesForCodegen => e.toCodegen() } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index 3d4fc75e83bb..e1d955f9d1ec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -17,13 +17,36 @@ package org.apache.spark.sql.internal +import org.scalatest.BeforeAndAfterAll + import org.apache.spark.sql.{QueryTest, Row, SparkSession, SQLContext} +import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.test.{SharedSQLContext, TestSQLContext} -class SQLConfSuite extends QueryTest with SharedSQLContext { +class SQLConfSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAll { + import testImplicits._ + private val testKey = "test.key.0" private val testVal = "test.val.0" + override def beforeAll() { + super.beforeAll() + sql("DROP TABLE IF EXISTS testData") + spark + .range(10) + .select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd) + .write + .saveAsTable("testData") + } + + override def afterAll(): Unit = { + try { + sql("DROP TABLE IF EXISTS testData") + } finally { + super.afterAll() + } + } + test("propagate from spark conf") { // We create a new context here to avoid order dependence with other tests that might call // clear(). @@ -219,4 +242,33 @@ class SQLConfSuite extends QueryTest with SharedSQLContext { } } + test("MAX_CASES_BRANCHES") { + val original = spark.conf.get(SQLConf.MAX_CASES_BRANCHES) + try { + val sql_one_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 END FROM testData" + val sql_two_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 ELSE 0 END FROM testData" + // When the value is zero, case when will not be part of wholestage codegen + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "0") + assert(!sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(!sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + + // When the value is one, if case when has one branch, we still enable codegen for case when. + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "1") + assert(sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(!sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "2") + assert(sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + } finally { + sql(s"set ${SQLConf.MAX_CASES_BRANCHES}=$original") + } + } + } From aaf5f2e4f2da231ccd9d5d7ddd4183c14c045f2b Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 28 May 2016 23:45:23 -0700 Subject: [PATCH 2/9] remove the useless comments. --- .../scala/org/apache/spark/sql/internal/SQLConfSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index e1d955f9d1ec..a70683eaf15e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -247,14 +247,13 @@ class SQLConfSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAl try { val sql_one_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 END FROM testData" val sql_two_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 ELSE 0 END FROM testData" - // When the value is zero, case when will not be part of wholestage codegen + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "0") assert(!sql(sql_one_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) assert(!sql(sql_two_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - // When the value is one, if case when has one branch, we still enable codegen for case when. spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "1") assert(sql(sql_one_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) From f351c104d363d97698bfa6ccd190b305c4334e78 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Sat, 28 May 2016 23:47:15 -0700 Subject: [PATCH 3/9] style fix. --- .../test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index a70683eaf15e..0e10663f2cd5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -266,7 +266,7 @@ class SQLConfSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAl assert(sql(sql_two_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) } finally { - sql(s"set ${SQLConf.MAX_CASES_BRANCHES}=$original") + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, s"$original") } } From ecc4318497de8c5e472aa663890bca4872d8109d Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 30 May 2016 16:35:33 -0700 Subject: [PATCH 4/9] fix --- .../spark/sql/hive/MetastoreRelation.scala | 2 +- .../sql/hive/execution/SQLQuerySuite.scala | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala index 9c820144aee1..5596a4470f3f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala @@ -160,7 +160,7 @@ private[hive] case class MetastoreRelation( val tPartition = new org.apache.hadoop.hive.metastore.api.Partition tPartition.setDbName(databaseName) tPartition.setTableName(tableName) - tPartition.setValues(p.spec.values.toList.asJava) + tPartition.setValues(partitionKeys.map(a => p.spec(a.name)).asJava) val sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor() tPartition.setSd(sd) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 4b51f021bfa0..a503d609a52d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -1537,6 +1537,24 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { assert(fs.exists(path), "This is an external table, so the data should not have been dropped") } + test("select partitioned table") { + sql( + s""" + |CREATE TABLE table_with_partition(c1 string) + |PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string) + """.stripMargin + ) + sql( + """ + |INSERT OVERWRITE TABLE table_with_partition + |PARTITION (p1='a',p2='b',p3='c',p4='d',p5='e') + |SELECT 'blarr' + """. stripMargin) + checkAnswer( + sql("SELECT p1, p2, p3, p4, p5, c1 FROM table_with_partition"), + Row("a", "b", "c", "d", "e", "blarr") :: Nil) + } + test("SPARK-14981: DESC not supported for sorting columns") { withTable("t") { val cause = intercept[ParseException] { From db67f8c8c216972965b28ce7fb13d40cb0f4cc54 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 30 May 2016 16:51:10 -0700 Subject: [PATCH 5/9] revert --- .../spark/sql/hive/MetastoreRelation.scala | 2 +- .../sql/hive/execution/SQLQuerySuite.scala | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala index 5596a4470f3f..9c820144aee1 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/MetastoreRelation.scala @@ -160,7 +160,7 @@ private[hive] case class MetastoreRelation( val tPartition = new org.apache.hadoop.hive.metastore.api.Partition tPartition.setDbName(databaseName) tPartition.setTableName(tableName) - tPartition.setValues(partitionKeys.map(a => p.spec(a.name)).asJava) + tPartition.setValues(p.spec.values.toList.asJava) val sd = new org.apache.hadoop.hive.metastore.api.StorageDescriptor() tPartition.setSd(sd) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index a503d609a52d..4b51f021bfa0 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -1537,24 +1537,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { assert(fs.exists(path), "This is an external table, so the data should not have been dropped") } - test("select partitioned table") { - sql( - s""" - |CREATE TABLE table_with_partition(c1 string) - |PARTITIONED BY (p1 string,p2 string,p3 string,p4 string,p5 string) - """.stripMargin - ) - sql( - """ - |INSERT OVERWRITE TABLE table_with_partition - |PARTITION (p1='a',p2='b',p3='c',p4='d',p5='e') - |SELECT 'blarr' - """. stripMargin) - checkAnswer( - sql("SELECT p1, p2, p3, p4, p5, c1 FROM table_with_partition"), - Row("a", "b", "c", "d", "e", "blarr") :: Nil) - } - test("SPARK-14981: DESC not supported for sorting columns") { withTable("t") { val cause = intercept[ParseException] { From 414e1163e2986035a9cf24dc7efca84eb0add66d Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 30 May 2016 17:02:48 -0700 Subject: [PATCH 6/9] address comments --- .../apache/spark/sql/catalyst/optimizer/Optimizer.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 4b31d7a10cbc..a779d19c3034 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -937,10 +937,14 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { */ case class OptimizeCodegen(conf: CatalystConf) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case e @ CaseWhen(branches, elseBranch) - if branches.size + elseBranch.size <= conf.maxCaseBranchesForCodegen => + case e: CaseWhen if canCodeGen(e) => e.toCodegen() } + + private def canCodeGen(e: CaseWhen): Boolean = { + val numBranches = e.branches.size + e.elseValue.size + numBranches <= conf.maxCaseBranchesForCodegen + } } /** From 4306c4fe0b741689bb0ff5349506707e8a7ec520 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 30 May 2016 21:48:10 -0700 Subject: [PATCH 7/9] address comments --- .../sql/catalyst/optimizer/Optimizer.scala | 3 +- .../spark/sql/internal/SQLConfSuite.scala | 75 ++++++++----------- 2 files changed, 32 insertions(+), 46 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index a779d19c3034..442e4bdf6d53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -937,8 +937,7 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { */ case class OptimizeCodegen(conf: CatalystConf) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case e: CaseWhen if canCodeGen(e) => - e.toCodegen() + case e: CaseWhen if canCodeGen(e) => e.toCodegen() } private def canCodeGen(e: CaseWhen): Boolean = { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index 0e10663f2cd5..03af64d5e497 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -17,36 +17,14 @@ package org.apache.spark.sql.internal -import org.scalatest.BeforeAndAfterAll - import org.apache.spark.sql.{QueryTest, Row, SparkSession, SQLContext} import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.test.{SharedSQLContext, TestSQLContext} -class SQLConfSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAll { - import testImplicits._ - +class SQLConfSuite extends QueryTest with SharedSQLContext { private val testKey = "test.key.0" private val testVal = "test.val.0" - override def beforeAll() { - super.beforeAll() - sql("DROP TABLE IF EXISTS testData") - spark - .range(10) - .select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd) - .write - .saveAsTable("testData") - } - - override def afterAll(): Unit = { - try { - sql("DROP TABLE IF EXISTS testData") - } finally { - super.afterAll() - } - } - test("propagate from spark conf") { // We create a new context here to avoid order dependence with other tests that might call // clear(). @@ -243,31 +221,40 @@ class SQLConfSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAl } test("MAX_CASES_BRANCHES") { + import testImplicits._ + val original = spark.conf.get(SQLConf.MAX_CASES_BRANCHES) try { - val sql_one_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 END FROM testData" - val sql_two_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 ELSE 0 END FROM testData" - - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "0") - assert(!sql(sql_one_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - assert(!sql(sql_two_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "1") - assert(sql(sql_one_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - assert(!sql(sql_two_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "2") - assert(sql(sql_one_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) - assert(sql(sql_two_branch_caseWhen) - .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + withTable("tab1") { + spark + .range(10) + .select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd) + .write + .saveAsTable("tab1") + + val sql_one_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 END FROM tab1" + val sql_two_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 ELSE 0 END FROM tab1" + + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "0") + assert(!sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(!sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "1") + assert(sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(!sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + + spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "2") + assert(sql(sql_one_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + assert(sql(sql_two_branch_caseWhen) + .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + } } finally { spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, s"$original") } } - } From b2849e8f514c1265f7c6199aba980e95b72aa7c2 Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Mon, 30 May 2016 22:13:05 -0700 Subject: [PATCH 8/9] address comments --- .../spark/sql/internal/SQLConfSuite.scala | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index 03af64d5e497..2cd3f475b6c0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -221,40 +221,31 @@ class SQLConfSuite extends QueryTest with SharedSQLContext { } test("MAX_CASES_BRANCHES") { - import testImplicits._ + withTable("tab1") { + spark.range(10).write.saveAsTable("tab1") + val sql_one_branch_caseWhen = "SELECT CASE WHEN id = 1 THEN 1 END FROM tab1" + val sql_two_branch_caseWhen = "SELECT CASE WHEN id = 1 THEN 1 ELSE 0 END FROM tab1" - val original = spark.conf.get(SQLConf.MAX_CASES_BRANCHES) - try { - withTable("tab1") { - spark - .range(10) - .select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd) - .write - .saveAsTable("tab1") - - val sql_one_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 END FROM tab1" - val sql_two_branch_caseWhen = "SELECT CASE WHEN a = 1 THEN 1 ELSE 0 END FROM tab1" - - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "0") + withSQLConf(SQLConf.MAX_CASES_BRANCHES.key -> "0") { assert(!sql(sql_one_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) assert(!sql(sql_two_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + } - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "1") + withSQLConf(SQLConf.MAX_CASES_BRANCHES.key -> "1") { assert(sql(sql_one_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) assert(!sql(sql_two_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) + } - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, "2") + withSQLConf(SQLConf.MAX_CASES_BRANCHES.key -> "2") { assert(sql(sql_one_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) assert(sql(sql_two_branch_caseWhen) .queryExecution.executedPlan.isInstanceOf[WholeStageCodegenExec]) } - } finally { - spark.conf.set(SQLConf.MAX_CASES_BRANCHES.key, s"$original") } } } From 9830e3175d06e2e381085d33a39352feb77d662c Mon Sep 17 00:00:00 2001 From: gatorsmile Date: Tue, 31 May 2016 07:10:11 -0700 Subject: [PATCH 9/9] address comments --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 442e4bdf6d53..3d9599231ffe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -937,10 +937,10 @@ object SimplifyConditionals extends Rule[LogicalPlan] with PredicateHelper { */ case class OptimizeCodegen(conf: CatalystConf) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transformAllExpressions { - case e: CaseWhen if canCodeGen(e) => e.toCodegen() + case e: CaseWhen if canCodegen(e) => e.toCodegen() } - private def canCodeGen(e: CaseWhen): Boolean = { + private def canCodegen(e: CaseWhen): Boolean = { val numBranches = e.branches.size + e.elseValue.size numBranches <= conf.maxCaseBranchesForCodegen }