From 9c258a5bbe19f1ea84aa6b9bab53f02093449a38 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 13 Nov 2016 04:40:13 +0900 Subject: [PATCH 1/2] Fix wholeTextFile tests in JavaAPISuite --- .../test/java/org/apache/spark/JavaAPISuite.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java index 533025ba83e72..3d14bdf8677ab 100644 --- a/core/src/test/java/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java @@ -46,6 +46,7 @@ import com.google.common.collect.Lists; import com.google.common.base.Throwables; import com.google.common.io.Files; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.DefaultCodec; @@ -1075,18 +1076,21 @@ public void wholeTextFiles() throws Exception { byte[] content2 = "spark is also easy to use.\n".getBytes(StandardCharsets.UTF_8); String tempDirName = tempDir.getAbsolutePath(); - Files.write(content1, new File(tempDirName + "/part-00000")); - Files.write(content2, new File(tempDirName + "/part-00001")); + URI path1 = new Path(tempDirName + "/part-00000").toUri(); + URI path2 = new Path(tempDirName + "/part-00001").toUri(); + + Files.write(content1, new File(path1.getPath())); + Files.write(content2, new File(path2.getPath())); Map container = new HashMap<>(); - container.put(tempDirName+"/part-00000", new Text(content1).toString()); - container.put(tempDirName+"/part-00001", new Text(content2).toString()); + container.put(path1.getPath(), new Text(content1).toString()); + container.put(path2.getPath(), new Text(content2).toString()); JavaPairRDD readRDD = sc.wholeTextFiles(tempDirName, 3); List> result = readRDD.collect(); for (Tuple2 res : result) { - assertEquals(res._2(), container.get(new URI(res._1()).getPath())); + assertEquals(res._2(), container.get(new Path(res._1()).toUri().getPath())); } } From e6b2a035ef63cc8b2d75769cbd32fedc9908b763 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 14 Nov 2016 21:09:36 +0900 Subject: [PATCH 2/2] Clean up a bit and leave a comment to note the path format --- .../test/java/org/apache/spark/JavaAPISuite.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java index 3d14bdf8677ab..7bebe0612f9a8 100644 --- a/core/src/test/java/org/apache/spark/JavaAPISuite.java +++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java @@ -20,7 +20,6 @@ import java.io.*; import java.nio.channels.FileChannel; import java.nio.ByteBuffer; -import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -1076,20 +1075,22 @@ public void wholeTextFiles() throws Exception { byte[] content2 = "spark is also easy to use.\n".getBytes(StandardCharsets.UTF_8); String tempDirName = tempDir.getAbsolutePath(); - URI path1 = new Path(tempDirName + "/part-00000").toUri(); - URI path2 = new Path(tempDirName + "/part-00001").toUri(); + String path1 = new Path(tempDirName, "part-00000").toUri().getPath(); + String path2 = new Path(tempDirName, "part-00001").toUri().getPath(); - Files.write(content1, new File(path1.getPath())); - Files.write(content2, new File(path2.getPath())); + Files.write(content1, new File(path1)); + Files.write(content2, new File(path2)); Map container = new HashMap<>(); - container.put(path1.getPath(), new Text(content1).toString()); - container.put(path2.getPath(), new Text(content2).toString()); + container.put(path1, new Text(content1).toString()); + container.put(path2, new Text(content2).toString()); JavaPairRDD readRDD = sc.wholeTextFiles(tempDirName, 3); List> result = readRDD.collect(); for (Tuple2 res : result) { + // Note that the paths from `wholeTextFiles` are in URI format on Windows, + // for example, file:/C:/a/b/c. assertEquals(res._2(), container.get(new Path(res._1()).toUri().getPath())); } }