Skip to content

Commit 268c13e

Browse files
committed
[SPARK-49054][SQL][3.5] Column default value should support current_* functions
### What changes were proposed in this pull request? This is a regression between Spark 3.5.0 and Spark 4. The following queries work on Spark 3.5.0 while fails on latest master branch: ``` CREATE TABLE test_current_user(i int, s string) USING parquet; ALTER TABLE test_current_user ALTER COLUMN s SET DEFAULT current_user() ``` ``` CREATE TABLE test_current_user(i int, s string default current_user()) USING parquet INSERT INTO test_current_user (i) VALUES ((0)); ``` This PR is to complete fixing this by eagerly executing finish-analysis and constant-folding rules before checking whether the expression is foldable and resolved. ### Why are the changes needed? Bug fix ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? New UTs ### Was this patch authored or co-authored using generative AI tooling? No Closes #47538 from gengliangwang/pickFinishAnlysis. Authored-by: Gengliang Wang <[email protected]> Signed-off-by: Gengliang Wang <[email protected]>
1 parent 75029e1 commit 268c13e

File tree

2 files changed

+30
-2
lines changed

2 files changed

+30
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.apache.spark.sql.catalyst.analysis._
2626
import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog}
2727
import org.apache.spark.sql.catalyst.expressions._
2828
import org.apache.spark.sql.catalyst.expressions.{Literal => ExprLiteral}
29-
import org.apache.spark.sql.catalyst.optimizer.ConstantFolding
29+
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, Optimizer}
3030
import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException}
3131
import org.apache.spark.sql.catalyst.plans.logical._
3232
import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
@@ -285,7 +285,9 @@ object ResolveDefaultColumns extends QueryErrorsBase with ResolveDefaultColumnsU
285285
val analyzer: Analyzer = DefaultColumnAnalyzer
286286
val analyzed = analyzer.execute(Project(Seq(Alias(parsed, colName)()), OneRowRelation()))
287287
analyzer.checkAnalysis(analyzed)
288-
ConstantFolding(analyzed)
288+
// Eagerly execute finish-analysis and constant-folding rules before checking whether the
289+
// expression is foldable and resolved.
290+
ConstantFolding(DefaultColumnOptimizer.FinishAnalysis(analyzed))
289291
} catch {
290292
case ex: AnalysisException =>
291293
throw QueryCompilationErrors.defaultValuesUnresolvedExprError(
@@ -452,6 +454,11 @@ object ResolveDefaultColumns extends QueryErrorsBase with ResolveDefaultColumnsU
452454
new CatalogManager(BuiltInFunctionCatalog, BuiltInFunctionCatalog.v1Catalog)) {
453455
}
454456

457+
/**
458+
* This is an Optimizer for convert default column expressions to foldable literals.
459+
*/
460+
object DefaultColumnOptimizer extends Optimizer(DefaultColumnAnalyzer.catalogManager)
461+
455462
/**
456463
* This is a FunctionCatalog for performing analysis using built-in functions only. It is a helper
457464
* for the DefaultColumnAnalyzer above.

sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,4 +215,25 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession {
215215
}
216216
}
217217
}
218+
219+
test("SPARK-49054: Create table with current_user() default") {
220+
val tableName = "test_current_user"
221+
val user = spark.sparkContext.sparkUser
222+
withTable(tableName) {
223+
sql(s"CREATE TABLE $tableName(i int, s string default current_user()) USING parquet")
224+
sql(s"INSERT INTO $tableName (i) VALUES ((0))")
225+
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user)))
226+
}
227+
}
228+
229+
test("SPARK-49054: Alter table with current_user() default") {
230+
val tableName = "test_current_user"
231+
val user = spark.sparkContext.sparkUser
232+
withTable(tableName) {
233+
sql(s"CREATE TABLE $tableName(i int, s string) USING parquet")
234+
sql(s"ALTER TABLE $tableName ALTER COLUMN s SET DEFAULT current_user()")
235+
sql(s"INSERT INTO $tableName (i) VALUES ((0))")
236+
checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user)))
237+
}
238+
}
218239
}

0 commit comments

Comments
 (0)