diff --git a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala index 5cd7397ea358f..0dab7c3876ebb 100644 --- a/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala +++ b/core/src/main/scala/org/apache/spark/internal/io/FileCommitProtocol.scala @@ -175,10 +175,19 @@ abstract class FileCommitProtocol extends Logging { /** * Specifies that a file should be deleted with the commit of this job. The default - * implementation deletes the file immediately. + * implementation deletes the file immediately or moves file to trash based on whether + * the trash feature is enabled. + * + * See https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/core-default.xml + * for the relevant trash configuration */ def deleteWithJob(fs: FileSystem, path: Path, recursive: Boolean): Boolean = { - fs.delete(path, recursive) + if (fs.getConf.getInt("fs.trash.interval", 0) > 0 && + Trash.moveToAppropriateTrash(fs, path, fs.getConf)) { + true + } else { + fs.delete(path, recursive); + } } /**