From 2834c30cbfeb1b1da74f0e3cd6a5e00bfdaa628f Mon Sep 17 00:00:00 2001 From: sidedoorleftroad Date: Sun, 5 Jul 2020 09:51:23 +0800 Subject: [PATCH 1/2] [SPARK-32172][CORE]Use createDirectory instead of mkdir --- .../scala/org/apache/spark/storage/DiskBlockManager.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index f2113947f6bf..c695f79445e0 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -18,6 +18,7 @@ package org.apache.spark.storage import java.io.{File, IOException} +import java.nio.file.Files import java.util.UUID import org.apache.spark.SparkConf @@ -69,9 +70,7 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea old } else { val newDir = new File(localDirs(dirId), "%02x".format(subDirId)) - if (!newDir.exists() && !newDir.mkdir()) { - throw new IOException(s"Failed to create local dir in $newDir.") - } + Files.createDirectory(newDir.toPath) subDirs(dirId)(subDirId) = newDir newDir } From d8ffb712b0820bfb4dbcae2c908a94a3a22c6397 Mon Sep 17 00:00:00 2001 From: sidedoorleftroad Date: Mon, 6 Jul 2020 15:57:17 +0800 Subject: [PATCH 2/2] Add file exists check --- .../scala/org/apache/spark/storage/DiskBlockManager.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index c695f79445e0..bf76eef443e8 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -70,7 +70,9 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea old } else { val newDir = new File(localDirs(dirId), "%02x".format(subDirId)) - Files.createDirectory(newDir.toPath) + if (!newDir.exists()) { + Files.createDirectory(newDir.toPath) + } subDirs(dirId)(subDirId) = newDir newDir }