Skip to content

Commit 09d2765

Browse files
szehon-hocloud-fan
authored andcommitted
[SPARK-53045][SQL] DESCRIBE EXTENDED should be resilient to corrupt metadata
### What changes were proposed in this pull request? A lot of methods called by DESCRIBE EXTENDED can throw exception if the CatalotTable metadata is not a certain way, in particular in the case of view which has strict requirements. This proposes to make DESCRIBE EXTENDED and CatalogTable.toString still able to return successfully in this case. ### Why are the changes needed? We see some issue where somehow there is some leftover View metadata on a table, and DESC EXTENDED and jobs attempting to call toString on the table both crash, even if the table is not a view ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing unit test ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#51753 from szehon-ho/view_query_output_cols. Authored-by: Szehon Ho <[email protected]> Signed-off-by: Wenchen Fan <[email protected]>
1 parent c9b85a2 commit 09d2765

File tree

1 file changed

+30
-25
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog

1 file changed

+30
-25
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.time.{ZoneId, ZoneOffset}
2222
import java.util.Date
2323

2424
import scala.collection.mutable
25+
import scala.util.Try
2526
import scala.util.control.NonFatal
2627

2728
import com.fasterxml.jackson.annotation.JsonInclude.Include
@@ -629,16 +630,6 @@ case class CatalogTable(
629630
if (lastAccessTime <= 0) JString("UNKNOWN")
630631
else JLong(lastAccessTime)
631632

632-
val viewQueryOutputColumns: JValue = {
633-
if (viewSchemaMode == SchemaEvolution) {
634-
JArray(schema.map(_.name).map(JString).toList)
635-
} else if (viewQueryColumnNames.nonEmpty) {
636-
JArray(viewQueryColumnNames.map(JString).toList)
637-
} else {
638-
JNull
639-
}
640-
}
641-
642633
val map = mutable.LinkedHashMap[String, JValue]()
643634

644635
if (identifier.catalog.isDefined) map += "Catalog" -> JString(identifier.catalog.get)
@@ -655,21 +646,35 @@ case class CatalogTable(
655646
}
656647
if (comment.isDefined) map += "Comment" -> JString(comment.get)
657648
if (collation.isDefined) map += "Collation" -> JString(collation.get)
658-
if (tableType == CatalogTableType.VIEW && viewText.isDefined) {
659-
map += "View Text" -> JString(viewText.get)
660-
}
661-
if (tableType == CatalogTableType.VIEW && viewOriginalText.isDefined) {
662-
map += "View Original Text" -> JString(viewOriginalText.get)
663-
}
664-
if (SQLConf.get.viewSchemaBindingEnabled && tableType == CatalogTableType.VIEW) {
665-
map += "View Schema Mode" -> JString(viewSchemaMode.toString)
666-
}
667-
if (viewCatalogAndNamespace.nonEmpty && tableType == CatalogTableType.VIEW) {
668-
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
669-
map += "View Catalog and Namespace" -> JString(viewCatalogAndNamespace.quoted)
670-
}
671-
if (viewQueryOutputColumns != JNull) {
672-
map += "View Query Output Columns" -> viewQueryOutputColumns
649+
650+
if (tableType == CatalogTableType.VIEW) {
651+
if (viewText.isDefined) {
652+
map += "View Text" -> JString(viewText.get)
653+
}
654+
if (viewOriginalText.isDefined) {
655+
map += "View Original Text" -> JString(viewOriginalText.get)
656+
}
657+
if (SQLConf.get.viewSchemaBindingEnabled) {
658+
val viewSchemaModeInfo = Try(viewSchemaMode.toString).getOrElse("UNKNOWN")
659+
map += "View Schema Mode" -> JString(viewSchemaModeInfo)
660+
}
661+
val viewCatalogAndNamespaceInfos = Try(viewCatalogAndNamespace).getOrElse(Seq.empty)
662+
if (viewCatalogAndNamespaceInfos.nonEmpty) {
663+
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
664+
map += "View Catalog and Namespace" -> JString(viewCatalogAndNamespaceInfos.quoted)
665+
}
666+
val viewQueryOutputColumns: JValue = Try {
667+
if (viewSchemaMode == SchemaEvolution) {
668+
JArray(schema.map(_.name).map(JString).toList)
669+
} else if (viewQueryColumnNames.nonEmpty) {
670+
JArray(viewQueryColumnNames.map(JString).toList)
671+
} else {
672+
JNull
673+
}
674+
}.getOrElse(JNull)
675+
if (viewQueryOutputColumns != JNull) {
676+
map += "View Query Output Columns" -> viewQueryOutputColumns
677+
}
673678
}
674679
if (tableProperties != JNull) map += "Table Properties" -> tableProperties
675680
stats.foreach { s =>

0 commit comments

Comments
 (0)