From 02686338e50f684b33610f29eec76283dabca4fc Mon Sep 17 00:00:00 2001 From: Zhongshuai Pei Date: Wed, 25 Nov 2015 09:33:49 +0800 Subject: [PATCH 1/4] can not use foreach({remove}) --- .../scala/org/apache/spark/util/ShutdownHookManager.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala index db4a8b304ec3e..1b953b8eefa7a 100644 --- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala +++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala @@ -57,7 +57,9 @@ private[spark] object ShutdownHookManager extends Logging { // Add a shutdown hook to delete the temp dirs when the JVM exits addShutdownHook(TEMP_DIR_SHUTDOWN_PRIORITY) { () => logInfo("Shutdown hook called") - shutdownDeletePaths.foreach { dirPath => + val arr = shutdownDeletePaths.toArray + for (i <- 0 until arr.length) { + val dirPath = arr(i) try { logInfo("Deleting directory " + dirPath) Utils.deleteRecursively(new File(dirPath)) From 4ee3a7907dfeffa4c29e27303bc00cf0b1ba7203 Mon Sep 17 00:00:00 2001 From: Zhongshuai Pei Date: Wed, 25 Nov 2015 10:57:58 +0800 Subject: [PATCH 2/4] Update ShutdownHookManager.scala --- .../scala/org/apache/spark/util/ShutdownHookManager.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala index 1b953b8eefa7a..ed8783922c33e 100644 --- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala +++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala @@ -57,9 +57,7 @@ private[spark] object ShutdownHookManager extends Logging { // Add a shutdown hook to delete the temp dirs when the JVM exits addShutdownHook(TEMP_DIR_SHUTDOWN_PRIORITY) { () => logInfo("Shutdown hook called") - val arr = shutdownDeletePaths.toArray - for (i <- 0 until arr.length) { - val dirPath = arr(i) + shutdownDeletePaths.toArray.foreach { dirPath => try { logInfo("Deleting directory " + dirPath) Utils.deleteRecursively(new File(dirPath)) From cea4cd34ca186be1a1733be438e0094ff228f2a9 Mon Sep 17 00:00:00 2001 From: Zhongshuai Pei Date: Wed, 25 Nov 2015 15:54:11 +0800 Subject: [PATCH 3/4] add a comment //we need to materialize the paths to delete because deleteRecursively removes items from //shutdownDeletePaths as we are traversing through it. --- .../main/scala/org/apache/spark/util/ShutdownHookManager.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala index ed8783922c33e..d74a5e32e753d 100644 --- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala +++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala @@ -57,6 +57,8 @@ private[spark] object ShutdownHookManager extends Logging { // Add a shutdown hook to delete the temp dirs when the JVM exits addShutdownHook(TEMP_DIR_SHUTDOWN_PRIORITY) { () => logInfo("Shutdown hook called") + //we need to materialize the paths to delete because deleteRecursively removes items from + //shutdownDeletePaths as we are traversing through it. shutdownDeletePaths.toArray.foreach { dirPath => try { logInfo("Deleting directory " + dirPath) From a3e7cb8f63749600ab9b33d338cd78a68a126a06 Mon Sep 17 00:00:00 2001 From: Zhongshuai Pei Date: Wed, 25 Nov 2015 15:59:29 +0800 Subject: [PATCH 4/4] add space after // --- .../scala/org/apache/spark/util/ShutdownHookManager.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala index d74a5e32e753d..4012dca3ecdf8 100644 --- a/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala +++ b/core/src/main/scala/org/apache/spark/util/ShutdownHookManager.scala @@ -57,8 +57,8 @@ private[spark] object ShutdownHookManager extends Logging { // Add a shutdown hook to delete the temp dirs when the JVM exits addShutdownHook(TEMP_DIR_SHUTDOWN_PRIORITY) { () => logInfo("Shutdown hook called") - //we need to materialize the paths to delete because deleteRecursively removes items from - //shutdownDeletePaths as we are traversing through it. + // we need to materialize the paths to delete because deleteRecursively removes items from + // shutdownDeletePaths as we are traversing through it. shutdownDeletePaths.toArray.foreach { dirPath => try { logInfo("Deleting directory " + dirPath)