Skip to content

[SPARK-29084][SQL][TESTS] Check method bytecode size in BenchmarkQueryTest #25788

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[SPARK-29084][SQL][TESTS] Check method bytecode size in BenchmarkQueryTest #25788

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
686a937
Fix
maropu Sep 14, 2019
9f32999
Fix
maropu Sep 21, 2019
93448be
Fix
maropu Sep 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions sql/core/src/test/scala/org/apache/spark/sql/BenchmarkQueryTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.spark.sql

import org.apache.spark.internal.config.Tests.IS_TESTING
import org.apache.spark.sql.catalyst.expressions.codegen.{CodeFormatter, CodeGenerator}
import org.apache.spark.sql.catalyst.expressions.codegen.{ByteCodeStats, CodeFormatter, CodeGenerator}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.{SparkPlan, WholeStageCodegenExec}
import org.apache.spark.sql.test.SharedSparkSession
Expand Down Expand Up @@ -48,7 +48,7 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
RuleExecutor.resetMetrics()
}

protected def checkGeneratedCode(plan: SparkPlan): Unit = {
protected def checkGeneratedCode(plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
val codegenSubtrees = new collection.mutable.HashSet[WholeStageCodegenExec]()
plan foreach {
case s: WholeStageCodegenExec =>
Expand All @@ -57,7 +57,7 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
}
codegenSubtrees.toSeq.foreach { subtree =>
val code = subtree.doCodeGen()._2
try {
val (_, ByteCodeStats(maxMethodCodeSize, _, _)) = try {
// Just check the generated code can be properly compiled
CodeGenerator.compile(code)
} catch {
Expand All @@ -72,6 +72,11 @@ abstract class BenchmarkQueryTest extends QueryTest with SharedSparkSession {
""".stripMargin
throw new Exception(msg, e)
}

assert(!checkMethodCodeSize ||
maxMethodCodeSize <= CodeGenerator.DEFAULT_JVM_HUGE_METHOD_LIMIT,
s"too long generated codes found in the WholeStageCodegenExec subtree (id=${subtree.id}) " +
s"and JIT optimization might not work:\n${subtree.treeString}")
}
}
}
10 changes: 8 additions & 2 deletions sql/core/src/test/scala/org/apache/spark/sql/TPCDSQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,19 @@ class TPCDSQuerySuite extends BenchmarkQueryTest with TPCDSSchema {
"q3", "q7", "q10", "q19", "q27", "q34", "q42", "q43", "q46", "q52", "q53", "q55", "q59",
"q63", "q65", "q68", "q73", "q79", "q89", "q98", "ss_max")

// List up the known queries having too large code in a generated function.
// A JIRA file for `modified-q3` is as follows;
// [SPARK-29128] Split predicate code in OR expressions
val blackListForMethodCodeSizeCheck = Set("modified-q3")
Copy link
Member

@dongjoon-hyun dongjoon-hyun Sep 21, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we have an IDed TODO to remove this, @maropu ? SPARK-29128 is the one?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Otherwise, we had better wait for the fix.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok


modifiedTPCDSQueries.foreach { name =>
val queryString = resourceToString(s"tpcds-modifiedQueries/$name.sql",
classLoader = Thread.currentThread().getContextClassLoader)
test(s"modified-$name") {
val testName = s"modified-$name"
test(testName) {
// check the plans can be properly generated
val plan = sql(queryString).queryExecution.executedPlan
checkGeneratedCode(plan)
checkGeneratedCode(plan, !blackListForMethodCodeSizeCheck.contains(testName))
}
}
}
5 changes: 3 additions & 2 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/LogicalPlanTagInSparkPlanSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ import org.apache.spark.sql.execution.window.WindowExec

class LogicalPlanTagInSparkPlanSuite extends TPCDSQuerySuite {

override protected def checkGeneratedCode(plan: SparkPlan): Unit = {
super.checkGeneratedCode(plan)
override protected def checkGeneratedCode(
plan: SparkPlan, checkMethodCodeSize: Boolean = true): Unit = {
super.checkGeneratedCode(plan, checkMethodCodeSize)
checkLogicalPlanTag(plan)
}

Expand Down