diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala index 2faf42028f3a..7d893872277d 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala @@ -545,7 +545,15 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat } private def getLocationFromStorageProps(table: CatalogTable): Option[String] = { - CaseInsensitiveMap(table.storage.properties).get("path") + val storageLoc = table.storage.locationUri.map(CatalogUtils.URIToString(_)) + val storageProp = CaseInsensitiveMap(table.storage.properties).get("path") + // storageProp == None is hive table + if (storageLoc.equals(storageProp) || storageProp == None) { + storageLoc + } else { + throw new AnalysisException(s"path in location ${storageLoc} " + + s"not equal to table prop path ${storageProp}, please use alter table in spark") + } } private def updateLocationInStorageProps( diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 270595b0011e..f3b6fe2d3bd4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -22,6 +22,7 @@ import java.net.URI import org.apache.hadoop.conf.Configuration import org.apache.spark.SparkConf +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.execution.QueryExecutionException @@ -202,4 +203,26 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { assert(alteredTable.provider === Some("foo")) }) } + + test("SPARK-31751: serde property `path` overwrites hive table property location") { + val catalog = newBasicCatalog() + val hiveTable = CatalogTable( + identifier = TableIdentifier("parq_alter", Some("db1")), + tableType = CatalogTableType.MANAGED, + storage = storageFormat, + schema = new StructType().add("col1", "int"), + provider = Some("parquet")) + catalog.createTable(hiveTable, ignoreIfExists = false) + val beforeAlterTable = externalCatalog.getTable("db1", "parq_alter") + assert(beforeAlterTable.storage.locationUri.toString.contains("parq_alter")) + + externalCatalog.client.runSqlHive( + "alter table db1.parq_alter rename to db1.parq_alter2") + + val e = intercept[AnalysisException]( + externalCatalog.getTable("db1", "parq_alter2") + ) + assert(e.getMessage.contains("not equal to table prop path") + && e.getMessage.contains("parq_alter2")) + } }