From 0c68926935eec48cb4b61175400694d8ff783564 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Tue, 20 Oct 2015 14:15:59 -0700 Subject: [PATCH 1/7] [SPARK-11188] Elide stacktraces in bin/spark-sql for AnalysisExceptions --- .../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 12 +++++++++++- .../spark/sql/hive/thriftserver/SparkSQLDriver.scala | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index b5073961a1c8..533d1b01b4c1 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -20,6 +20,8 @@ package org.apache.spark.sql.hive.thriftserver import java.io._ import java.util.{ArrayList => JArrayList, Locale} +import org.apache.spark.sql.AnalysisException + import scala.collection.JavaConverters._ import jline.console.ConsoleReader @@ -308,7 +310,15 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { ret = rc.getResponseCode if (ret != 0) { - console.printError(rc.getErrorMessage()) + // For analysis exception, only the error is printed out to the console. + rc.getException() match { + case e : AnalysisException => + console.printError( + s"""Failed with exception ${e.getClass.getName}: ${e.getMessage}""") + case _ => { + console.printError(rc.getErrorMessage()) + } + } driver.close() return ret } diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala index 2619286afc14..b1a918195a2f 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala @@ -65,7 +65,7 @@ private[hive] class SparkSQLDriver( } catch { case cause: Throwable => logError(s"Failed in [$command]", cause) - new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), null) + new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), null, cause) } } From cea5b05ccc35a9bb803d2c6e1de6ea330018bd13 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Wed, 21 Oct 2015 22:10:04 -0700 Subject: [PATCH 2/7] Implement code review comments --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 7 ++----- .../org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 7 +++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 533d1b01b4c1..90b3ac369ad5 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -313,11 +313,8 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { // For analysis exception, only the error is printed out to the console. rc.getException() match { case e : AnalysisException => - console.printError( - s"""Failed with exception ${e.getClass.getName}: ${e.getMessage}""") - case _ => { - console.printError(rc.getErrorMessage()) - } + console.printError(s"""Error in query: ${e.getMessage}""") + case _ => console.printError(rc.getErrorMessage()) } driver.close() return ret diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 76d1591a235c..43da89db40d3 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -219,4 +219,11 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { -> "OK" ) } + + test("SPARK-11188: Analysis error reporting") { + runCliWithin(2.minute)( + "select * from nonexistent_table;" + -> "Error in query: no such table nonexistent_table;" + ) + } } From 02ff04156d00036fa78daec7a67528e59dd85523 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Thu, 22 Oct 2015 01:46:04 -0700 Subject: [PATCH 3/7] Fix up the expected output as per recent change to error message --- .../org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 43da89db40d3..fa7a0e2f3c7d 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -220,10 +220,10 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { ) } - test("SPARK-11188: Analysis error reporting") { + test("SPARK-11188 Analysis error reporting") { runCliWithin(2.minute)( "select * from nonexistent_table;" - -> "Error in query: no such table nonexistent_table;" + -> "Error in query: Table Not Found: nonexistent_table;" ) } } From 0e298a81383d426db4e980f69a18fe00f4cf2d07 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Thu, 22 Oct 2015 01:52:05 -0700 Subject: [PATCH 4/7] Fix up the expected output as per recent change to error message --- .../scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index fa7a0e2f3c7d..3abdcc62f147 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -223,7 +223,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { test("SPARK-11188 Analysis error reporting") { runCliWithin(2.minute)( "select * from nonexistent_table;" - -> "Error in query: Table Not Found: nonexistent_table;" + -> "Error in query: Table not Found: nonexistent_table;" ) } } From cb06db0a4d766eba03f2b21699d22ba9bf469b9a Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Fri, 23 Oct 2015 23:15:10 -0700 Subject: [PATCH 5/7] fix test failures --- .../spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala | 5 +++-- .../org/apache/spark/sql/hive/thriftserver/CliSuite.scala | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index 90b3ac369ad5..62e912c69abc 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -300,6 +300,7 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { driver.init() val out = sessionState.out + val err = sessionState.err val start: Long = System.currentTimeMillis() if (sessionState.getIsVerbose) { out.println(cmd) @@ -313,8 +314,8 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { // For analysis exception, only the error is printed out to the console. rc.getException() match { case e : AnalysisException => - console.printError(s"""Error in query: ${e.getMessage}""") - case _ => console.printError(rc.getErrorMessage()) + err.println(s"""Error in query: ${e.getMessage}""") + case _ => err.println(rc.getErrorMessage()) } driver.close() return ret diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index 3abdcc62f147..a683f2aaf406 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -223,7 +223,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { test("SPARK-11188 Analysis error reporting") { runCliWithin(2.minute)( "select * from nonexistent_table;" - -> "Error in query: Table not Found: nonexistent_table;" + -> "Error in query: Table not found: nonexistent_table;" ) } } From bbfeaed85954cf76285fe60e89f29fa9e3b76c11 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Tue, 27 Oct 2015 01:45:50 -0700 Subject: [PATCH 6/7] Supressing logging to console --- .../hive/thriftserver/SparkSQLDriver.scala | 25 ++++++++++++++++--- .../sql/hive/thriftserver/CliSuite.scala | 7 +++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala index b1a918195a2f..5f2c22a8688a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala @@ -18,6 +18,8 @@ package org.apache.spark.sql.hive.thriftserver import java.util.{Arrays, ArrayList => JArrayList, List => JList} +import org.apache.log4j.LogManager +import org.apache.spark.sql.AnalysisException import scala.collection.JavaConverters._ @@ -63,9 +65,26 @@ private[hive] class SparkSQLDriver( tableSchema = getResultSetSchema(execution) new CommandProcessorResponse(0) } catch { - case cause: Throwable => - logError(s"Failed in [$command]", cause) - new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), null, cause) + case ae: AnalysisException => + // On analysis exception we will supress printing of the exception + // on to console. We do that by removing the console appender. If + // Logging is setup to log to both console and file , we will still + // log the error to the file. + val appender = LogManager.getRootLogger().getAppender("console") + if (appender != null) { + LogManager.getRootLogger().removeAppender("console") + } + + logError(s"Failed in [$command]", ae) + + // Restore the console appender. + if (appender != null) { + LogManager.getRootLogger().addAppender(appender) + } + new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(ae), null, ae) + case cause: Throwable => + logError(s"Failed in [$command]", cause) + new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(cause), null, cause) } } diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index a683f2aaf406..3fa5c8528b60 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -58,7 +58,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { * @param timeout maximum time for the commands to complete * @param extraArgs any extra arguments * @param errorResponses a sequence of strings whose presence in the stdout of the forked process - * is taken as an immediate error condition. That is: if a line beginning + * is taken as an immediate error condition. That is: if a line containing * with one of these strings is found, fail the test immediately. * The default value is `Seq("Error:")` * @@ -104,7 +104,7 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { } } else { errorResponses.foreach { r => - if (line.startsWith(r)) { + if (line.contains(r)) { foundAllExpectedAnswers.tryFailure( new RuntimeException(s"Failed with error line '$line'")) } @@ -221,7 +221,8 @@ class CliSuite extends SparkFunSuite with BeforeAndAfter with Logging { } test("SPARK-11188 Analysis error reporting") { - runCliWithin(2.minute)( + runCliWithin(timeout = 2.minute, + errorResponses = Seq("AnalysisException"))( "select * from nonexistent_table;" -> "Error in query: Table not found: nonexistent_table;" ) From df1a3fa0159415771c62c738b32dbb47a37fa58b Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Wed, 28 Oct 2015 14:39:32 -0700 Subject: [PATCH 7/7] Code review comments from Michael --- .../sql/hive/thriftserver/SparkSQLDriver.scala | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala index 5f2c22a8688a..f1ec7238520a 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLDriver.scala @@ -66,21 +66,7 @@ private[hive] class SparkSQLDriver( new CommandProcessorResponse(0) } catch { case ae: AnalysisException => - // On analysis exception we will supress printing of the exception - // on to console. We do that by removing the console appender. If - // Logging is setup to log to both console and file , we will still - // log the error to the file. - val appender = LogManager.getRootLogger().getAppender("console") - if (appender != null) { - LogManager.getRootLogger().removeAppender("console") - } - - logError(s"Failed in [$command]", ae) - - // Restore the console appender. - if (appender != null) { - LogManager.getRootLogger().addAppender(appender) - } + logDebug(s"Failed in [$command]", ae) new CommandProcessorResponse(1, ExceptionUtils.getStackTrace(ae), null, ae) case cause: Throwable => logError(s"Failed in [$command]", cause)