Skip to content

Commit 18ec016

Browse files
committed
add one AnalysisBarrier and address comments
1 parent 0e4d2e1 commit 18ec016

File tree

5 files changed

+17
-17
lines changed

5 files changed

+17
-17
lines changed

sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.apache.spark.annotation.InterfaceStability
2626
import org.apache.spark.sql.catalyst.TableIdentifier
2727
import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedRelation}
2828
import org.apache.spark.sql.catalyst.catalog._
29-
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoTable, LogicalPlan}
29+
import org.apache.spark.sql.catalyst.plans.logical.{AnalysisBarrier, InsertIntoTable, LogicalPlan}
3030
import org.apache.spark.sql.execution.SQLExecution
3131
import org.apache.spark.sql.execution.command.DDLUtils
3232
import org.apache.spark.sql.execution.datasources.{CreateTable, DataSource, LogicalRelation}
@@ -264,7 +264,7 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
264264
sparkSession = df.sparkSession,
265265
className = source,
266266
partitionColumns = partitioningColumns.getOrElse(Nil),
267-
options = extraOptions.toMap).planForWriting(mode, df.logicalPlan)
267+
options = extraOptions.toMap).planForWriting(mode, AnalysisBarrier(df.logicalPlan))
268268
}
269269
}
270270
}

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.plans._
2727
import org.apache.spark.sql.catalyst.plans.logical._
2828
import org.apache.spark.sql.catalyst.plans.physical._
2929
import org.apache.spark.sql.execution.columnar.{InMemoryRelation, InMemoryTableScanExec}
30-
import org.apache.spark.sql.execution.command.{DataWritingCommand, _}
30+
import org.apache.spark.sql.execution.command._
3131
import org.apache.spark.sql.execution.exchange.ShuffleExchangeExec
3232
import org.apache.spark.sql.execution.joins.{BuildLeft, BuildRight, BuildSide}
3333
import org.apache.spark.sql.execution.streaming._

sql/core/src/main/scala/org/apache/spark/sql/execution/command/DataWritingCommand.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration
2222
import org.apache.spark.SparkContext
2323
import org.apache.spark.sql.{Row, SparkSession}
2424
import org.apache.spark.sql.catalyst.expressions.Attribute
25-
import org.apache.spark.sql.catalyst.plans.logical.{AnalysisBarrier, Command, LogicalPlan}
25+
import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan}
2626
import org.apache.spark.sql.execution.SparkPlan
2727
import org.apache.spark.sql.execution.datasources.BasicWriteJobStatsTracker
2828
import org.apache.spark.sql.execution.datasources.FileFormatWriter

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,4 +2150,17 @@ class HiveDDLSuite
21502150
assert(e.message.contains("LOAD DATA input path does not exist"))
21512151
}
21522152
}
2153+
2154+
test("SPARK-22252: FileFormatWriter should respect the input query schema in HIVE") {
2155+
withTable("t1", "t2", "t3", "t4") {
2156+
spark.range(1).select('id as 'col1, 'id as 'col2).write.saveAsTable("t1")
2157+
spark.sql("select COL1, COL2 from t1").write.format("hive").saveAsTable("t2")
2158+
checkAnswer(spark.table("t2"), Row(0, 0))
2159+
2160+
// Test picking part of the columns when writing.
2161+
spark.range(1).select('id, 'id as 'col1, 'id as 'col2).write.saveAsTable("t3")
2162+
spark.sql("select COL1, COL2 from t3").write.format("hive").saveAsTable("t4")
2163+
checkAnswer(spark.table("t4"), Row(0, 0))
2164+
}
2165+
}
21532166
}

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1970,19 +1970,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
19701970
}
19711971
}
19721972

1973-
test("SPARK-22252: FileFormatWriter should respect the input query schema in HIVE") {
1974-
withTable("t1", "t2", "t3", "t4") {
1975-
spark.range(1).select('id as 'col1, 'id as 'col2).write.saveAsTable("t1")
1976-
spark.sql("select COL1, COL2 from t1").write.format("hive").saveAsTable("t2")
1977-
checkAnswer(spark.table("t2"), Row(0, 0))
1978-
1979-
// Test picking part of the columns when writing.
1980-
spark.range(1).select('id, 'id as 'col1, 'id as 'col2).write.saveAsTable("t3")
1981-
spark.sql("select COL1, COL2 from t3").write.format("hive").saveAsTable("t4")
1982-
checkAnswer(spark.table("t4"), Row(0, 0))
1983-
}
1984-
}
1985-
19861973
test("Auto alias construction of get_json_object") {
19871974
val df = Seq(("1", """{"f1": "value1", "f5": 5.23}""")).toDF("key", "jstring")
19881975
val expectedMsg = "Cannot create a table having a column whose name contains commas " +

0 commit comments

Comments
 (0)