From b78d0e0d513c992bd3180db8d24e972ff63c8b64 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Thu, 19 Sep 2019 07:08:11 +0900 Subject: [PATCH 1/2] [MINOR][SQL] Always log generated code in case of compile error on generated code in UT --- .../expressions/codegen/CodeGenerator.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 1d010ff5abfc..024c7c358c83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1369,20 +1369,28 @@ object CodeGenerator extends Logging { case e: InternalCompilerException => val msg = s"failed to compile: $e" logError(msg, e) - val maxLines = SQLConf.get.loggingMaxLinesForCodegen - logInfo(s"\n${CodeFormatter.format(code, maxLines)}") + logGeneratedCode(code) throw new InternalCompilerException(msg, e) case e: CompileException => val msg = s"failed to compile: $e" logError(msg, e) - val maxLines = SQLConf.get.loggingMaxLinesForCodegen - logInfo(s"\n${CodeFormatter.format(code, maxLines)}") + logGeneratedCode(code) throw new CompileException(msg, e.getLocation) } (evaluator.getClazz().getConstructor().newInstance().asInstanceOf[GeneratedClass], codeStats) } + private def logGeneratedCode(code: CodeAndComment): Unit = { + val maxLines = SQLConf.get.loggingMaxLinesForCodegen + if (Utils.isTesting) { + // log as ERROR so that it can be always logged in UT + logError(s"\n${CodeFormatter.format(code, maxLines)}") + } else { + logInfo(s"\n${CodeFormatter.format(code, maxLines)}") + } + } + /** * Returns the bytecode statistics (max method bytecode size, max constant pool size, and * # of inner classes) of generated classes by inspecting Janino classes. From d836138d2599996bf0b429cab7d6706547d4b2bc Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Thu, 19 Sep 2019 09:53:34 +0900 Subject: [PATCH 2/2] Reflect review comment --- .../spark/sql/catalyst/expressions/codegen/CodeGenerator.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala index 024c7c358c83..68ddec9fc8d0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala @@ -1384,7 +1384,6 @@ object CodeGenerator extends Logging { private def logGeneratedCode(code: CodeAndComment): Unit = { val maxLines = SQLConf.get.loggingMaxLinesForCodegen if (Utils.isTesting) { - // log as ERROR so that it can be always logged in UT logError(s"\n${CodeFormatter.format(code, maxLines)}") } else { logInfo(s"\n${CodeFormatter.format(code, maxLines)}")