From 6c25d6b92c5564ad455b9d065fc1e983c3ff08ea Mon Sep 17 00:00:00 2001 From: nirajguavus Date: Wed, 23 Apr 2014 16:23:23 -0700 Subject: [PATCH 1/4] Fixed bug for Jira Issue# 1527 Details: rootDirs in DiskBlockManagerSuite doesn't get full path from rootDir0, rootDir1 --- .../scala/org/apache/spark/storage/DiskBlockManagerSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala index 9b29e2a8a55d..17fbef7ac661 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala @@ -26,6 +26,7 @@ import com.google.common.io.Files import org.scalatest.{BeforeAndAfterEach, FunSuite} import org.apache.spark.SparkConf +import java.nio.file.Files class DiskBlockManagerSuite extends FunSuite with BeforeAndAfterEach { private val testConf = new SparkConf(false) @@ -33,7 +34,7 @@ class DiskBlockManagerSuite extends FunSuite with BeforeAndAfterEach { rootDir0.deleteOnExit() val rootDir1 = Files.createTempDir() rootDir1.deleteOnExit() - val rootDirs = rootDir0.getName + "," + rootDir1.getName + val rootDirs = rootDir0.getCanonicalPath + "," + rootDir1.getCanonicalPath println("Created root dirs: " + rootDirs) // This suite focuses primarily on consolidation features, From 53ed683214ebbe24e8af80a1bcfc35b4685bb73d Mon Sep 17 00:00:00 2001 From: nirajsuthar Date: Thu, 24 Apr 2014 11:25:15 -0700 Subject: [PATCH 2/4] Jira 1527 rootDirs in DiskBlockManagerSuite doesn't get full path from rootDir0, rootDir1 --- .../scala/org/apache/spark/storage/DiskBlockManagerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala index 17fbef7ac661..57c34f18761f 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskBlockManagerSuite.scala @@ -26,7 +26,7 @@ import com.google.common.io.Files import org.scalatest.{BeforeAndAfterEach, FunSuite} import org.apache.spark.SparkConf -import java.nio.file.Files + class DiskBlockManagerSuite extends FunSuite with BeforeAndAfterEach { private val testConf = new SparkConf(false) From 59e1ad11afdd6bfa167a305169e89289e7f5b413 Mon Sep 17 00:00:00 2001 From: nirajsuthar Date: Thu, 24 Apr 2014 19:39:08 -0700 Subject: [PATCH 3/4] fixed it to use absolute path instead of relative Its same issue as Jira 1527 - fixed it to use absolute path instead of relative --- .../main/scala/org/apache/spark/broadcast/HttpBroadcast.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala index 29372f16f2ca..685e91d9e809 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala @@ -213,7 +213,7 @@ private[spark] object HttpBroadcast extends Logging { SparkEnv.get.blockManager.master.removeBroadcast(id, removeFromDriver, blocking) if (removeFromDriver) { val file = getFile(id) - files.remove(file.toString) + files.remove(file.getCanonicalPath) deleteBroadcastFile(file) } } @@ -229,7 +229,7 @@ private[spark] object HttpBroadcast extends Logging { val (file, time) = (entry.getKey, entry.getValue) if (time < cleanupTime) { iterator.remove() - deleteBroadcastFile(new File(file.toString)) + deleteBroadcastFile(new File(file.getCanonicalPath)) } } } From d35b0717ceabcec15a3c328e76993e985f281085 Mon Sep 17 00:00:00 2001 From: nirajsuthar Date: Thu, 24 Apr 2014 23:14:33 -0700 Subject: [PATCH 4/4] should take abs path. SPARK-1623. Broadcast cleaner should use getCanonicalPath when deleting files by name --- .../main/scala/org/apache/spark/broadcast/HttpBroadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala index 685e91d9e809..5fb38ff75e30 100644 --- a/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala +++ b/core/src/main/scala/org/apache/spark/broadcast/HttpBroadcast.scala @@ -229,7 +229,7 @@ private[spark] object HttpBroadcast extends Logging { val (file, time) = (entry.getKey, entry.getValue) if (time < cleanupTime) { iterator.remove() - deleteBroadcastFile(new File(file.getCanonicalPath)) + deleteBroadcastFile(new File(file)) } } }