From c52010a3fa291e85f75f862d2d75363e8505fcc7 Mon Sep 17 00:00:00 2001 From: Imran Rashid Date: Fri, 16 Nov 2018 16:45:13 -0600 Subject: [PATCH 1/2] [SPARK-26094][CORE][STREAMING] createNonEcFile creates parent dirs. We explicitly avoid files with hdfs erasure coding for the streaming WAL and for event logs, as hdfs EC does not support all relevant apis. However, the new builder api used has different semantics -- it does not create parent dirs, and it does not resolve relative paths. This updates createNonEcFile to have similar semantics to the old api. --- .../scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index 5979151345415..d911827d78c53 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -471,7 +471,11 @@ object SparkHadoopUtil { try { // Use reflection as this uses apis only avialable in hadoop 3 val builderMethod = fs.getClass().getMethod("createFile", classOf[Path]) - val builder = builderMethod.invoke(fs, path) + // the builder api does not resolve relative paths, nor does it create parent dirs, while + // the old api does. + fs.mkdirs(path.getParent()) + val qualifiedPath = fs.makeQualified(path) + val builder = builderMethod.invoke(fs, qualifiedPath) val builderCls = builder.getClass() // this may throw a NoSuchMethodException if the path is not on hdfs val replicateMethod = builderCls.getMethod("replicate") From 891e37cd371350906035cfbf93c618035a4293b1 Mon Sep 17 00:00:00 2001 From: Imran Rashid Date: Mon, 3 Dec 2018 14:29:04 -0600 Subject: [PATCH 2/2] add error handling on mkdirs --- .../main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index d911827d78c53..7bb38ccf6f995 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -473,7 +473,9 @@ object SparkHadoopUtil { val builderMethod = fs.getClass().getMethod("createFile", classOf[Path]) // the builder api does not resolve relative paths, nor does it create parent dirs, while // the old api does. - fs.mkdirs(path.getParent()) + if (!fs.mkdirs(path.getParent())) { + throw new IOException(s"Failed to create parents of $path") + } val qualifiedPath = fs.makeQualified(path) val builder = builderMethod.invoke(fs, qualifiedPath) val builderCls = builder.getClass()