|
20 | 20 | import java.io.*; |
21 | 21 | import java.nio.channels.FileChannel; |
22 | 22 | import java.nio.ByteBuffer; |
23 | | -import java.net.URI; |
24 | 23 | import java.nio.charset.StandardCharsets; |
25 | 24 | import java.util.ArrayList; |
26 | 25 | import java.util.Arrays; |
|
46 | 45 | import com.google.common.collect.Lists; |
47 | 46 | import com.google.common.base.Throwables; |
48 | 47 | import com.google.common.io.Files; |
| 48 | +import org.apache.hadoop.fs.Path; |
49 | 49 | import org.apache.hadoop.io.IntWritable; |
50 | 50 | import org.apache.hadoop.io.Text; |
51 | 51 | import org.apache.hadoop.io.compress.DefaultCodec; |
@@ -1075,18 +1075,23 @@ public void wholeTextFiles() throws Exception { |
1075 | 1075 | byte[] content2 = "spark is also easy to use.\n".getBytes(StandardCharsets.UTF_8); |
1076 | 1076 |
|
1077 | 1077 | String tempDirName = tempDir.getAbsolutePath(); |
1078 | | - Files.write(content1, new File(tempDirName + "/part-00000")); |
1079 | | - Files.write(content2, new File(tempDirName + "/part-00001")); |
| 1078 | + String path1 = new Path(tempDirName, "part-00000").toUri().getPath(); |
| 1079 | + String path2 = new Path(tempDirName, "part-00001").toUri().getPath(); |
| 1080 | + |
| 1081 | + Files.write(content1, new File(path1)); |
| 1082 | + Files.write(content2, new File(path2)); |
1080 | 1083 |
|
1081 | 1084 | Map<String, String> container = new HashMap<>(); |
1082 | | - container.put(tempDirName+"/part-00000", new Text(content1).toString()); |
1083 | | - container.put(tempDirName+"/part-00001", new Text(content2).toString()); |
| 1085 | + container.put(path1, new Text(content1).toString()); |
| 1086 | + container.put(path2, new Text(content2).toString()); |
1084 | 1087 |
|
1085 | 1088 | JavaPairRDD<String, String> readRDD = sc.wholeTextFiles(tempDirName, 3); |
1086 | 1089 | List<Tuple2<String, String>> result = readRDD.collect(); |
1087 | 1090 |
|
1088 | 1091 | for (Tuple2<String, String> res : result) { |
1089 | | - assertEquals(res._2(), container.get(new URI(res._1()).getPath())); |
| 1092 | + // Note that the paths from `wholeTextFiles` are in URI format on Windows, |
| 1093 | + // for example, file:/C:/a/b/c. |
| 1094 | + assertEquals(res._2(), container.get(new Path(res._1()).toUri().getPath())); |
1090 | 1095 | } |
1091 | 1096 | } |
1092 | 1097 |
|
|
0 commit comments