From 48344fbcd1eb16b233217b782a1d8722e8a1da5c Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 24 Oct 2019 11:53:31 +0800 Subject: [PATCH 1/3] [SPARK-29530][SQL] Make SQLConf in SQL parse process thread safe --- .../spark/sql/catalyst/parser/ParseDriver.scala | 12 ++++++------ .../apache/spark/sql/execution/SparkSqlParser.scala | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index 7d8cb1f18b4b..c04983ee9a6d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.types.{DataType, StructType} /** * Base SQL parsing infrastructure. */ -abstract class AbstractSqlParser extends ParserInterface with Logging { +abstract class AbstractSqlParser(conf: SQLConf) extends ParserInterface with Logging { /** Creates/Resolves DataType for a given SQL string. */ override def parseDataType(sqlText: String): DataType = parse(sqlText) { parser => @@ -84,14 +84,14 @@ abstract class AbstractSqlParser extends ParserInterface with Logging { val lexer = new SqlBaseLexer(new UpperCaseCharStream(CharStreams.fromString(command))) lexer.removeErrorListeners() lexer.addErrorListener(ParseErrorListener) - lexer.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced + lexer.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced val tokenStream = new CommonTokenStream(lexer) val parser = new SqlBaseParser(tokenStream) parser.addParseListener(PostProcessor) parser.removeErrorListeners() parser.addErrorListener(ParseErrorListener) - parser.legacy_setops_precedence_enbled = SQLConf.get.setOpsPrecedenceEnforced + parser.legacy_setops_precedence_enbled = conf.setOpsPrecedenceEnforced try { try { @@ -125,13 +125,13 @@ abstract class AbstractSqlParser extends ParserInterface with Logging { /** * Concrete SQL parser for Catalyst-only SQL statements. */ -class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser { +class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) { val astBuilder = new AstBuilder(conf) } /** For test-only. */ -object CatalystSqlParser extends AbstractSqlParser { - val astBuilder = new AstBuilder(new SQLConf()) +object CatalystSqlParser extends AbstractSqlParser(new SQLConf) { + val astBuilder = new AstBuilder(new SQLConf) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 89cb63784c0f..95206e8ef182 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -39,7 +39,7 @@ import org.apache.spark.sql.types.StructType /** * Concrete parser for Spark SQL statements. */ -class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser { +class SparkSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) { val astBuilder = new SparkSqlAstBuilder(conf) private val substitutor = new VariableSubstitution(conf) From bc36fb5b3ea2a9771c621639d9a612ecc8ad1446 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 24 Oct 2019 11:57:44 +0800 Subject: [PATCH 2/3] Update ParseDriver.scala --- .../org/apache/spark/sql/catalyst/parser/ParseDriver.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index c04983ee9a6d..96e45cbcbc71 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -130,8 +130,8 @@ class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) { } /** For test-only. */ -object CatalystSqlParser extends AbstractSqlParser(new SQLConf) { - val astBuilder = new AstBuilder(new SQLConf) +object CatalystSqlParser extends AbstractSqlParser(new SQLConf()) { + val astBuilder = new AstBuilder(new SQLConf()) } /** From 8dc40474121e1de8c7312ff8166ef56dd067cdf4 Mon Sep 17 00:00:00 2001 From: angerszhu Date: Thu, 24 Oct 2019 16:57:58 +0800 Subject: [PATCH 3/3] Update ParseDriver.scala --- .../org/apache/spark/sql/catalyst/parser/ParseDriver.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala index 96e45cbcbc71..454861a9b7e9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala @@ -130,8 +130,8 @@ class CatalystSqlParser(conf: SQLConf) extends AbstractSqlParser(conf) { } /** For test-only. */ -object CatalystSqlParser extends AbstractSqlParser(new SQLConf()) { - val astBuilder = new AstBuilder(new SQLConf()) +object CatalystSqlParser extends AbstractSqlParser(SQLConf.get) { + val astBuilder = new AstBuilder(SQLConf.get) } /**