Skip to content

Commit e210385

Browse files
fenzhuGitHub Enterprise
authored andcommitted
[CARMEL-6365] Don't store data in quota free folder when create temp table (#1148)
1 parent 3617606 commit e210385

File tree

1 file changed

+14
-1
lines changed

1 file changed

+14
-1
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/FileFormatWriter.scala

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,10 +117,23 @@ object FileFormatWriter extends Logging {
117117
// scalastyle:on argcount
118118
val writeJobUuid = UUID.randomUUID().toString
119119

120+
val insertIntoTempTable = {
121+
def checkPath(p: Path): Boolean = {
122+
val uri = p.toUri
123+
uri != null && uri.getAuthority != null && uri.getPath != null
124+
}
125+
val destPath = new Path(outputSpec.outputPath)
126+
val tempTablePath = sparkSession.sessionState.catalog.defaultTempTableParentPath
127+
checkPath(destPath) && checkPath(tempTablePath) &&
128+
destPath.toUri.getAuthority.equalsIgnoreCase(tempTablePath.toUri.getAuthority) &&
129+
destPath.toUri.getPath.startsWith(tempTablePath.toUri.getPath)
130+
}
131+
120132
val quotaFreeDir = sparkSession.sessionState.conf.quotaFreeDir
121133
// Support to unset the quota free configuration with:
122134
// --conf spark.sql.source.output.quotaFreeDir=NONE;
123-
if (quotaFreeDir.isDefined && !quotaFreeDir.get.trim.toUpperCase().equals("NONE")) {
135+
if (!insertIntoTempTable && quotaFreeDir.isDefined
136+
&& !quotaFreeDir.get.trim.toUpperCase().equals("NONE")) {
124137
val hasValidQuotaFreeDir = Try { new Path(quotaFreeDir.get) }.isSuccess
125138
if (hasValidQuotaFreeDir) {
126139
// temp output directory: quotaFreeDir/applicationId_sessionId/uuid/

0 commit comments

Comments
 (0)