From 2bb4fc3546959eba93f628afa92a368e22a9574f Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 00:43:32 -0700 Subject: [PATCH 01/10] Select correct spark submit and a test case can't work on windows for FsHistoryProviderSuite --- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 7 ++++++- .../spark/deploy/history/FsHistoryProviderSuite.scala | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 271897699201b..037a0afc5f078 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -575,8 +575,13 @@ class SparkSubmitSuite // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly. private def runSparkSubmit(args: Seq[String]): Unit = { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) + val sparkSubmit = if (Utils.isWindows) { + Seq(new File("..\\bin\\spark-submit.cmd").getAbsolutePath) ++ args + } else { + Seq("./bin/spark-submit") ++ args + } val process = Utils.executeCommand( - Seq("./bin/spark-submit") ++ args, + sparkSubmit, new File(sparkHome), Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome)) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala index 39c5857b13451..8e2e9c10419bf 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala @@ -127,6 +127,8 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc } test("SPARK-3697: ignore directories that cannot be read.") { + // setReadable doesn't work on windows for directories + assume(!Utils.isWindows) val logFile1 = newLogFile("new1", None, inProgress = false) writeFile(logFile1, true, None, SparkListenerApplicationStart("app1-1", Some("app1-1"), 1L, "test", None), From bd28500436a5e932a4ea40cdb5690c21d2e2e06e Mon Sep 17 00:00:00 2001 From: Tao LI Date: Mon, 25 Apr 2016 01:10:00 -0700 Subject: [PATCH 02/10] Circular buffer on windows. --- .../test/scala/org/apache/spark/util/UtilsSuite.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index 4aa4854c36f3a..fe52b88b72f5f 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -686,7 +686,14 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { val stream = new java.io.PrintStream(buffer, true, "UTF-8") // scalastyle:off println - stream.println("test circular test circular test circular test circular test circular") + // Note: println will append '\r\n' in windows, + // so there will be one more byte comparing to Unix/Linux. + if (Utils.isWindows) { + stream.print("test circular test circular test circular test circular test circular\n") + } else { + // scalastyle:off println + stream.println("test circular test circular test circular test circular test circular") + } // scalastyle:on println assert(buffer.toString === "t circular test circular\n") } From 13880425d4caa4ffaf2f724720239c070b7308e5 Mon Sep 17 00:00:00 2001 From: Tao LI Date: Mon, 25 Apr 2016 01:25:12 -0700 Subject: [PATCH 03/10] Java bug on windows. --- .../org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala index cb88a1c83c999..24b0014188bcd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala @@ -67,6 +67,8 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSQLContext with } test("CREATE TEMPORARY TABLE AS SELECT based on the file without write permission") { + // setWritable(boolean) doesn't work on Windows + assume(!Utils.isWindows) val childPath = new File(path.toString, "child") path.mkdir() childPath.createNewFile() From f98c9a9e7538eb6e9ecd1e650e4db7d016a8e395 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 04:22:33 -0700 Subject: [PATCH 04/10] fix windows issue --- .../test/scala/org/apache/spark/storage/DiskStoreSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala index 9ed5016510d56..df37a7953ced0 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala @@ -22,10 +22,12 @@ import java.util.Arrays import org.apache.spark.{SparkConf, SparkFunSuite} import org.apache.spark.util.io.ChunkedByteBuffer +import org.apache.spark.util.Utils class DiskStoreSuite extends SparkFunSuite { test("reads of memory-mapped and non memory-mapped files are equivalent") { + assume(!Utils.isWindows) val confKey = "spark.storage.memoryMapThreshold" // Create a non-trivial (not all zeros) byte array From cd12232650e02d236d26b4ee38c9aced4113bf24 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 25 Apr 2016 19:12:07 -0700 Subject: [PATCH 05/10] Check symlink for win --- core/src/main/scala/org/apache/spark/util/Utils.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index ea49991493fd7..accbaf50aa151 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -23,7 +23,7 @@ import java.net._ import java.nio.ByteBuffer import java.nio.channels.Channels import java.nio.charset.StandardCharsets -import java.nio.file.Files +import java.nio.file.{Files, Paths} import java.util.{Locale, Properties, Random, UUID} import java.util.concurrent._ import javax.net.ssl.HttpsURLConnection @@ -36,7 +36,6 @@ import scala.io.Source import scala.reflect.ClassTag import scala.util.Try import scala.util.control.{ControlThrowable, NonFatal} - import com.google.common.io.{ByteStreams, Files => GFiles} import com.google.common.net.InetAddresses import org.apache.commons.lang3.SystemUtils @@ -47,7 +46,6 @@ import org.apache.log4j.PropertyConfigurator import org.eclipse.jetty.util.MultiException import org.json4s._ import org.slf4j.Logger - import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging @@ -947,7 +945,9 @@ private[spark] object Utils extends Logging { */ def isSymlink(file: File): Boolean = { if (file == null) throw new NullPointerException("File must not be null") - if (isWindows) return false + if (isWindows) { + return Files.isSymbolicLink(Paths.get(file.toURI)) + } val fileInCanonicalDir = if (file.getParent() == null) { file } else { From 61305e031dfd3da2550d95654063c290cb08e2c4 Mon Sep 17 00:00:00 2001 From: Tao LI Date: Tue, 3 May 2016 10:35:27 +0800 Subject: [PATCH 06/10] Fix style --- core/src/main/scala/org/apache/spark/util/Utils.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index accbaf50aa151..7151dcddc0a6b 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -36,6 +36,7 @@ import scala.io.Source import scala.reflect.ClassTag import scala.util.Try import scala.util.control.{ControlThrowable, NonFatal} + import com.google.common.io.{ByteStreams, Files => GFiles} import com.google.common.net.InetAddresses import org.apache.commons.lang3.SystemUtils @@ -46,6 +47,7 @@ import org.apache.log4j.PropertyConfigurator import org.eclipse.jetty.util.MultiException import org.json4s._ import org.slf4j.Logger + import org.apache.spark._ import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging From a8f3a7028a2056f50a466629b1bc2dfb5d9642e5 Mon Sep 17 00:00:00 2001 From: Tao LI Date: Tue, 3 May 2016 12:53:37 +0800 Subject: [PATCH 07/10] Fix subject to PR comments --- .../src/main/scala/org/apache/spark/util/Utils.scala | 12 +----------- .../org/apache/spark/deploy/SparkSubmitSuite.scala | 7 +------ .../org/apache/spark/storage/DiskStoreSuite.scala | 2 ++ .../scala/org/apache/spark/util/UtilsSuite.scala | 7 +------ 4 files changed, 5 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 7151dcddc0a6b..241025e828cc1 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -946,17 +946,7 @@ private[spark] object Utils extends Logging { * Check to see if file is a symbolic link. */ def isSymlink(file: File): Boolean = { - if (file == null) throw new NullPointerException("File must not be null") - if (isWindows) { - return Files.isSymbolicLink(Paths.get(file.toURI)) - } - val fileInCanonicalDir = if (file.getParent() == null) { - file - } else { - new File(file.getParentFile().getCanonicalFile(), file.getName()) - } - - !fileInCanonicalDir.getCanonicalFile().equals(fileInCanonicalDir.getAbsoluteFile()) + return Files.isSymbolicLink(Paths.get(file.toURI)) } /** diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 037a0afc5f078..271897699201b 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -575,13 +575,8 @@ class SparkSubmitSuite // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly. private def runSparkSubmit(args: Seq[String]): Unit = { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) - val sparkSubmit = if (Utils.isWindows) { - Seq(new File("..\\bin\\spark-submit.cmd").getAbsolutePath) ++ args - } else { - Seq("./bin/spark-submit") ++ args - } val process = Utils.executeCommand( - sparkSubmit, + Seq("./bin/spark-submit") ++ args, new File(sparkHome), Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome)) diff --git a/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala b/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala index df37a7953ced0..9e6b02b9eac4d 100644 --- a/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala +++ b/core/src/test/scala/org/apache/spark/storage/DiskStoreSuite.scala @@ -27,6 +27,8 @@ import org.apache.spark.util.Utils class DiskStoreSuite extends SparkFunSuite { test("reads of memory-mapped and non memory-mapped files are equivalent") { + // It will cause error when we tried to re-open the filestore and the + // memory-mapped byte buffer tot he file has not been GC on Windows. assume(!Utils.isWindows) val confKey = "spark.storage.memoryMapThreshold" diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala index fe52b88b72f5f..0ed866a74b584 100644 --- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala @@ -688,12 +688,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging { // scalastyle:off println // Note: println will append '\r\n' in windows, // so there will be one more byte comparing to Unix/Linux. - if (Utils.isWindows) { - stream.print("test circular test circular test circular test circular test circular\n") - } else { - // scalastyle:off println - stream.println("test circular test circular test circular test circular test circular") - } + stream.print("test circular test circular test circular test circular test circular\n") // scalastyle:on println assert(buffer.toString === "t circular test circular\n") } From 18fbda57c373ab5c5e2995a4b6b24527ae5089ba Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Mon, 2 May 2016 22:55:02 -0700 Subject: [PATCH 08/10] Revert sparksubmit change --- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 271897699201b..037a0afc5f078 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -575,8 +575,13 @@ class SparkSubmitSuite // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly. private def runSparkSubmit(args: Seq[String]): Unit = { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) + val sparkSubmit = if (Utils.isWindows) { + Seq(new File("..\\bin\\spark-submit.cmd").getAbsolutePath) ++ args + } else { + Seq("./bin/spark-submit") ++ args + } val process = Utils.executeCommand( - Seq("./bin/spark-submit") ++ args, + sparkSubmit, new File(sparkHome), Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome)) From 1d6345e74f3e3f255efd3cf0ab44b5dd7f4fd9a7 Mon Sep 17 00:00:00 2001 From: "U-FAREAST\\tl" Date: Wed, 4 May 2016 20:14:34 -0700 Subject: [PATCH 09/10] Minor code cleanup --- .../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 037a0afc5f078..89085e56346a5 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -575,13 +575,13 @@ class SparkSubmitSuite // NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly. private def runSparkSubmit(args: Seq[String]): Unit = { val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")) - val sparkSubmit = if (Utils.isWindows) { - Seq(new File("..\\bin\\spark-submit.cmd").getAbsolutePath) ++ args + val sparkSubmitFile = if (Utils.isWindows) { + new File("..\\bin\\spark-submit.cmd") } else { - Seq("./bin/spark-submit") ++ args + new File("./bin/spark-submit.cmd") } val process = Utils.executeCommand( - sparkSubmit, + Seq(sparkSubmitFile.getCanonicalPath) ++ args, new File(sparkHome), Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome)) From 080ceb5287c8819e00a5ddae503004276d9be1ff Mon Sep 17 00:00:00 2001 From: Tao LI Date: Thu, 5 May 2016 11:45:54 +0800 Subject: [PATCH 10/10] Fix path issue of SparkSubmitSuite on Linux --- .../test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 89085e56346a5..26e047f6b3686 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -578,7 +578,7 @@ class SparkSubmitSuite val sparkSubmitFile = if (Utils.isWindows) { new File("..\\bin\\spark-submit.cmd") } else { - new File("./bin/spark-submit.cmd") + new File("../bin/spark-submit") } val process = Utils.executeCommand( Seq(sparkSubmitFile.getCanonicalPath) ++ args,