From dc502493c8c5cde03ba4dc1ce8391e176c583267 Mon Sep 17 00:00:00 2001 From: Ruslan Shestopalyuk Date: Wed, 6 Sep 2017 17:24:43 +0200 Subject: [PATCH] Fix DiskBlockManager crashing when root local folder has been removed --- .../org/apache/spark/storage/DiskBlockManager.scala | 2 +- .../apache/spark/storage/DiskBlockManagerSuite.scala | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala index 3d43e3c367aac..5c886d72c7b53 100644 --- a/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala +++ b/core/src/main/scala/org/apache/spark/storage/DiskBlockManager.scala @@ -66,7 +66,7 @@ private[spark] class DiskBlockManager(conf: SparkConf, deleteFilesOnStop: Boolea old } else { val newDir = new File(localDirs(dirId), "%02x".format(subDirId)) - if (!newDir.exists() && !newDir.mkdir()) { + if (!newDir.exists() && !newDir.mkdirs()) { throw new IOException(s"Failed to create local dir in $newDir.") } subDirs(dirId)(subDirId) = newDir diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala index 7859b0bba2b48..3ddd1101223b0 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala @@ -79,6 +79,16 @@ class DiskBlockManagerSuite extends SparkFunSuite with BeforeAndAfterEach with B assert(diskBlockManager.getAllBlocks.toSet === ids.toSet) } + test("working correctly if local dirs are deleted") { + // DisckBlockManager's local dirs deletion can happen externally, + // for example when they are located in the '/tmp' folder (which is default) + diskBlockManager.localDirs.foreach(Utils.deleteRecursively(_)) + val blockId = new TestBlockId("test") + val newFile = diskBlockManager.getFile(blockId) + writeToFile(newFile, 10) + assert(diskBlockManager.containsBlock(blockId)) + } + def writeToFile(file: File, numBytes: Int) { val writer = new FileWriter(file, true) for (i <- 0 until numBytes) writer.write(i)