From a6d48558111b6968919a692f6b2d8a9095fc64a4 Mon Sep 17 00:00:00 2001 From: fjh100456 Date: Thu, 13 Sep 2018 19:28:12 +0800 Subject: [PATCH] [SPARK-25404][SQL] Staging path may not on the expected place when table path contains the stagingDir string ## What changes were proposed in this pull request? Fix Staging path. ## How was this patch tested? Manual test. --- .../apache/spark/sql/hive/execution/SaveAsHiveFile.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala index 078968ed0145f..f28f21a5fc012 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala @@ -217,12 +217,7 @@ private[hive] trait SaveAsHiveFile extends DataWritingCommand { val inputPathUri: URI = inputPath.toUri val inputPathName: String = inputPathUri.getPath val fs: FileSystem = inputPath.getFileSystem(hadoopConf) - var stagingPathName: String = - if (inputPathName.indexOf(stagingDir) == -1) { - new Path(inputPathName, stagingDir).toString - } else { - inputPathName.substring(0, inputPathName.indexOf(stagingDir) + stagingDir.length) - } + var stagingPathName: String = new Path(inputPathName, stagingDir).toString // SPARK-20594: This is a walk-around fix to resolve a Hive bug. Hive requires that the // staging directory needs to avoid being deleted when users set hive.exec.stagingdir