Skip to content

[SPARK-19242][SQL] SHOW CREATE TABLE should generate new syntax to create hive table #16600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[SPARK-19242][SQL] SHOW CREATE TABLE should generate new syntax to create hive table #16600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -761,6 +761,9 @@ case class AlterTableSetLocationCommand(

object DDLUtils {
val HIVE_PROVIDER = "hive"
val HIVE_SERDE_OPTION = "serde"
val HIVE_INPUT_FORMAT_OPTION = "inputFormat"
val HIVE_OUTPUT_FORMAT_OPTION = "outputFormat"

def isHiveTable(table: CatalogTable): Boolean = {
table.provider.isDefined && table.provider.get.toLowerCase == HIVE_PROVIDER
Expand Down
117 changes: 60 additions & 57 deletions sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -789,55 +789,50 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
)

override def run(sparkSession: SparkSession): Seq[Row] = {
val catalog = sparkSession.sessionState.catalog
val tableMetadata = catalog.getTableMetadata(table)

// TODO: unify this after we unify the CREATE TABLE syntax for hive serde and data source table.
val stmt = if (DDLUtils.isDatasourceTable(tableMetadata)) {
showCreateDataSourceTable(tableMetadata)
} else {
showCreateHiveTable(tableMetadata)
}
val tableMeta = sparkSession.sessionState.catalog.getTableMetadata(table)
val tableName = tableMeta.identifier.quotedString

Seq(Row(stmt))
}

private def showCreateHiveTable(metadata: CatalogTable): String = {
def reportUnsupportedError(features: Seq[String]): Unit = {
if (tableMeta.unsupportedFeatures.nonEmpty) {
throw new AnalysisException(
s"Failed to execute SHOW CREATE TABLE against table/view ${metadata.identifier}, " +
s"Failed to execute SHOW CREATE TABLE against table/view $tableName, " +
"which is created by Hive and uses the following unsupported feature(s)\n" +
features.map(" - " + _).mkString("\n")
)
tableMeta.unsupportedFeatures.map(" - " + _).mkString("\n"))
}

if (metadata.unsupportedFeatures.nonEmpty) {
reportUnsupportedError(metadata.unsupportedFeatures)
}
val stmt = if (tableMeta.tableType == VIEW) {
val builder = StringBuilder.newBuilder
builder ++= s"CREATE VIEW $tableName"

val builder = StringBuilder.newBuilder
if (tableMeta.schema.nonEmpty) {
builder ++= tableMeta.schema.map(_.name).mkString("(", ", ", ")")
}

val tableTypeString = metadata.tableType match {
case EXTERNAL => " EXTERNAL TABLE"
case VIEW => " VIEW"
case MANAGED => " TABLE"
builder ++= s" AS\n${tableMeta.viewText.get}"
builder.toString
} else if (DDLUtils.isHiveTable(tableMeta) && tableMeta.properties.nonEmpty) {
Copy link
Contributor Author

@cloud-fan cloud-fan Jan 16, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this condition is true most of the time, because Hive metastore will generate some table properties. https://issues.apache.org/jira/browse/SPARK-19241 is tracking it

// If table properties are not empty, this Hive table was probably created via legacy Hive
// syntax, and we have to generate CREATE TABLE statement using legacy syntax, as the
// official syntax doesn't support table properties.
showCreateTableWithLegacySyntax(tableMeta)
} else {
showCreateTable(tableMeta)
}

builder ++= s"CREATE$tableTypeString ${table.quotedString}"
Seq(Row(stmt))
}

if (metadata.tableType == VIEW) {
if (metadata.schema.nonEmpty) {
builder ++= metadata.schema.map(_.name).mkString("(", ", ", ")")
}
builder ++= metadata.viewText.mkString(" AS\n", "", "\n")
} else {
showHiveTableHeader(metadata, builder)
showHiveTableNonDataColumns(metadata, builder)
showHiveTableStorageInfo(metadata, builder)
showHiveTableProperties(metadata, builder)
}
private def showCreateTableWithLegacySyntax(metadata: CatalogTable): String = {
val builder = StringBuilder.newBuilder

builder.toString()
val isExternal = if (metadata.tableType == EXTERNAL) " EXTERNAL" else ""
builder ++= s"CREATE$isExternal TABLE ${metadata.identifier.quotedString}"

showHiveTableHeader(metadata, builder)
showHiveTableNonDataColumns(metadata, builder)
showHiveTableStorageInfo(metadata, builder)
showHiveTableProperties(metadata, builder)

builder.toString
}

private def showHiveTableHeader(metadata: CatalogTable, builder: StringBuilder): Unit = {
Expand Down Expand Up @@ -915,47 +910,55 @@ case class ShowCreateTableCommand(table: TableIdentifier) extends RunnableComman
}
}

private def showCreateDataSourceTable(metadata: CatalogTable): String = {
private def showCreateTable(metadata: CatalogTable): String = {
val builder = StringBuilder.newBuilder

builder ++= s"CREATE TABLE ${table.quotedString} "
showDataSourceTableDataColumns(metadata, builder)
showDataSourceTableOptions(metadata, builder)
showDataSourceTableNonDataColumns(metadata, builder)
builder ++= s"CREATE TABLE ${metadata.identifier.quotedString} "

showColumns(metadata, builder)
showDataSourceOptions(metadata, builder)
showPartitioningAndBucketing(metadata, builder)

if (metadata.tableType == EXTERNAL) {
builder ++= s"LOCATION '${metadata.storage.locationUri.get}'\n"
}

metadata.comment.foreach { comment =>
builder ++= s"COMMENT '${escapeSingleQuotedString(comment)}'"
}

builder.toString()
}

private def showDataSourceTableDataColumns(
metadata: CatalogTable, builder: StringBuilder): Unit = {
private def showColumns(metadata: CatalogTable, builder: StringBuilder): Unit = {
val columns = metadata.schema.fields.map(f => s"${quoteIdentifier(f.name)} ${f.dataType.sql}")
builder ++= columns.mkString("(", ", ", ")\n")
}

private def showDataSourceTableOptions(metadata: CatalogTable, builder: StringBuilder): Unit = {
private def showDataSourceOptions(metadata: CatalogTable, builder: StringBuilder): Unit = {
builder ++= s"USING ${metadata.provider.get}\n"

val dataSourceOptions = metadata.storage.properties.map {
case (key, value) => s"${quoteIdentifier(key)} '${escapeSingleQuotedString(value)}'"
} ++ metadata.storage.locationUri.flatMap { location =>
if (metadata.tableType == MANAGED) {
// If it's a managed table, omit PATH option. Spark SQL always creates external table
// when the table creation DDL contains the PATH option.
None
} else {
Some(s"path '${escapeSingleQuotedString(location)}'")
}
}

val hiveOptions = if (DDLUtils.isHiveTable(metadata)) {
Seq(
s"${DDLUtils.HIVE_SERDE_OPTION} '${metadata.storage.serde.get}'",
s"${DDLUtils.HIVE_INPUT_FORMAT_OPTION} '${metadata.storage.inputFormat.get}'",
s"${DDLUtils.HIVE_OUTPUT_FORMAT_OPTION} '${metadata.storage.outputFormat.get}'")
} else {
Seq.empty[String]
}

if (dataSourceOptions.nonEmpty) {
builder ++= "OPTIONS (\n"
builder ++= dataSourceOptions.mkString(" ", ",\n ", "\n")
builder ++= (dataSourceOptions ++ hiveOptions).mkString(" ", ",\n ", "\n")
builder ++= ")\n"
}
}

private def showDataSourceTableNonDataColumns(
metadata: CatalogTable, builder: StringBuilder): Unit = {
private def showPartitioningAndBucketing(metadata: CatalogTable, builder: StringBuilder): Unit = {
val partCols = metadata.partitionColumnNames
if (partCols.nonEmpty) {
builder ++= s"PARTITIONED BY ${partCols.mkString("(", ", ", ")")}\n"
Expand Down
7 changes: 4 additions & 3 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveOptions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.spark.sql.hive.execution

import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
import org.apache.spark.sql.execution.command.DDLUtils

/**
* Options for the Hive data source. Note that rule `DetermineHiveSerde` will extract Hive
Expand Down Expand Up @@ -87,9 +88,9 @@ object HiveOptions {
}

val FILE_FORMAT = newOption("fileFormat")
val INPUT_FORMAT = newOption("inputFormat")
val OUTPUT_FORMAT = newOption("outputFormat")
val SERDE = newOption("serde")
val INPUT_FORMAT = newOption(DDLUtils.HIVE_INPUT_FORMAT_OPTION)
val OUTPUT_FORMAT = newOption(DDLUtils.HIVE_OUTPUT_FORMAT_OPTION)
val SERDE = newOption(DDLUtils.HIVE_SERDE_OPTION)

// A map from the public delimiter option keys to the underlying Hive serde property keys.
val delimiterOptions = Map(
Expand Down
4 changes: 1 addition & 3 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/ShowCreateTableSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -338,9 +338,7 @@ class ShowCreateTableSuite extends QueryTest with SQLTestUtils with TestHiveSing
"totalSize",
"totalNumberFiles",
"maxFileSize",
"minFileSize",
// EXTERNAL is not non-deterministic, but it is filtered out for external tables.
"EXTERNAL"
"minFileSize"
)

table.copy(
Expand Down