diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8c7702efd47f..4b0d110b077f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3818,6 +3818,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES = + buildConf("spark.sql.legacy.descNamespaceRedactProperties") + .internal() + .doc("When set to false, redact sensitive information in the result of DESC NAMESPACE " + + "EXTENDED. If set to true, it restores the legacy behavior that this sensitive " + + "information was included in the output.") + .version("3.4.0") + .booleanConf + .createWithDefault(false) + /** * Holds information about keys that have been deprecated. * diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala index 5cdcf33d6cde..19b737d7d80d 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala @@ -190,6 +190,8 @@ case class DescribeDatabaseCommand( val propertiesStr = if (properties.isEmpty) { "" + } else if (SQLConf.get.getConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES)) { + properties.toSeq.sortBy(_._1).mkString("(", ", ", ")") } else { conf.redactOptions(properties).toSeq.sortBy(_._1).mkString("(", ", ", ")") } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala index 75c12ea4201d..950511e16c89 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DescribeNamespaceExec.scala @@ -23,6 +23,7 @@ import scala.collection.mutable.ArrayBuffer import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.connector.catalog.{CatalogV2Util, SupportsNamespaces} +import org.apache.spark.sql.internal.SQLConf /** * Physical plan node for describing a namespace. @@ -48,6 +49,8 @@ case class DescribeNamespaceExec( val propertiesStr = if (properties.isEmpty) { "" + } else if (SQLConf.get.getConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES)) { + properties.toSeq.sortBy(_._1).mkString("(", ", ", ")") } else { conf.redactOptions(properties.toMap).toSeq.sortBy(_._1).mkString("(", ", ", ")") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala index 645399b90261..3f1108f379e0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.command.v2 import org.apache.spark.sql.Row import org.apache.spark.sql.connector.catalog.SupportsNamespaces import org.apache.spark.sql.execution.command +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType} import org.apache.spark.util.Utils @@ -30,28 +31,49 @@ class DescribeNamespaceSuite extends command.DescribeNamespaceSuiteBase with Com override def notFoundMsgPrefix: String = "Namespace" test("DescribeNamespace using v2 catalog") { - withNamespace(s"$catalog.ns1.ns2") { - sql( - s""" - | CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 - | COMMENT 'test namespace' - | LOCATION '/tmp/ns_test' - | WITH DBPROPERTIES (password = 'password') + withSQLConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES.key -> "false") { + withNamespace(s"$catalog.ns1.ns2") { + sql( + s""" + | CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 + | COMMENT 'test namespace' + | LOCATION '/tmp/ns_test' + | WITH DBPROPERTIES (password = 'password') """.stripMargin) - val descriptionDf = sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.ns1.ns2") - assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === - Seq( - ("info_name", StringType), - ("info_value", StringType) - )) - val description = descriptionDf.collect() - assert(description === Seq( - Row("Namespace Name", "ns2"), - Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), - Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), - Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName()), - Row("Properties", "((password,*********(redacted)))")) - ) + val descriptionDf = sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.ns1.ns2") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("info_name", StringType), + ("info_value", StringType) + )) + val description = descriptionDf.collect() + assert(description === Seq( + Row("Namespace Name", "ns2"), + Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), + Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), + Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName()), + Row("Properties", "((password,*********(redacted)))")) + ) + } + } + withSQLConf(SQLConf.LEGACY_DESC_NAMESPACE_REDACT_PROPERTIES.key -> "true") { + withNamespace(s"$catalog.ns1.ns2") { + sql(s"CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 COMMENT " + + "'test namespace' LOCATION '/tmp/ns_test'") + val descriptionDf = sql(s"DESCRIBE NAMESPACE $catalog.ns1.ns2") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("info_name", StringType), + ("info_value", StringType) + )) + val description = descriptionDf.collect() + assert(description === Seq( + Row("Namespace Name", "ns2"), + Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), + Row(SupportsNamespaces.PROP_LOCATION.capitalize, "file:/tmp/ns_test"), + Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName())) + ) + } } }