From 13ba8035d4e79fda1f5837485249062266b16e56 Mon Sep 17 00:00:00 2001 From: Ramesh Thangarajan Date: Wed, 11 Mar 2020 01:18:34 -0700 Subject: [PATCH 1/2] HADOOP-16769 LocalDirAllocator to provide diagnostics when file creation fails --- .../apache/hadoop/fs/LocalDirAllocator.java | 22 +++++++++++++++---- .../hadoop/fs/TestLocalDirAllocator.java | 16 ++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index 5f266a7b82555..26e5e303c34cf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -393,6 +393,8 @@ public Path getLocalPathForWrite(String pathStr, long size, Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; + long maxCapacity = 0; + String errorText = null; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { @@ -441,9 +443,16 @@ public Path getLocalPathForWrite(String pathStr, long size, int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc); while (numDirsSearched < numDirs) { long capacity = ctx.dirDF[dirNum].getAvailable(); + if (capacity > maxCapacity) { + maxCapacity = capacity; + } if (capacity > size) { - returnPath = - createPath(ctx.localDirs[dirNum], pathStr, checkWrite); + try { + returnPath = createPath(ctx.localDirs[dirNum], pathStr, + checkWrite); + } catch (Exception e) { + errorText = e.getMessage(); + } if (returnPath != null) { ctx.getAndIncrDirNumLastAccessed(numDirsSearched); break; @@ -459,8 +468,13 @@ public Path getLocalPathForWrite(String pathStr, long size, } //no path found - throw new DiskErrorException("Could not find any valid local " + - "directory for " + pathStr); + String newErrorText = "Could not find any valid local directory for " + + pathStr + " with requested size " + size + + " as the max capacity in any directory is " + maxCapacity; + if (errorText != null) { + newErrorText = newErrorText + " due to " + errorText; + } + throw new DiskErrorException(newErrorText); } /** Creates a file on the local FS. Pass size as diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index acda898ea1342..ef54e7dd2c8ea 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -26,6 +26,7 @@ import java.util.NoSuchElementException; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.Shell; @@ -532,4 +533,19 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception { } } + /** + * Test to check the LocalDirAllocation for the less space HADOOP-16769. + * + * @throws Exception + */ + @Test(timeout = 30000) + public void testGetLocalPathForWriteForLessSpace() throws Exception { + String dir0 = buildBufferDir(ROOT, 0); + String dir1 = buildBufferDir(ROOT, 1); + conf.set(CONTEXT, dir0 + "," + dir1); + LambdaTestUtils.intercept(DiskErrorException.class, "as the max capacity" + + " in any directory is", "Expect a DiskErrorException.", () -> + dirAllocator.getLocalPathForWrite("p1/x", Long.MAX_VALUE-1, conf)); + } + } From 84a43a99fe0aeedb99771e780b4e0876ff4862ee Mon Sep 17 00:00:00 2001 From: Ramesh Thangarajan Date: Thu, 14 May 2020 15:45:05 -0700 Subject: [PATCH 2/2] HADOOP-16769 LocalDirAllocator to provide diagnostics when file creation fails --- .../java/org/apache/hadoop/fs/LocalDirAllocator.java | 9 +++++++-- .../java/org/apache/hadoop/fs/TestLocalDirAllocator.java | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index 26e5e303c34cf..e1b0d048ea76a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -395,6 +395,7 @@ public Path getLocalPathForWrite(String pathStr, long size, int numDirsSearched = 0; long maxCapacity = 0; String errorText = null; + IOException diskException = null; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { @@ -450,8 +451,12 @@ public Path getLocalPathForWrite(String pathStr, long size, try { returnPath = createPath(ctx.localDirs[dirNum], pathStr, checkWrite); - } catch (Exception e) { + } catch (IOException e) { errorText = e.getMessage(); + diskException = e; + if (LOG.isDebugEnabled()) { + LOG.debug("DiskException caught for dir " + ctx.localDirs[dirNum].toString(), e); + } } if (returnPath != null) { ctx.getAndIncrDirNumLastAccessed(numDirsSearched); @@ -474,7 +479,7 @@ public Path getLocalPathForWrite(String pathStr, long size, if (errorText != null) { newErrorText = newErrorText + " due to " + errorText; } - throw new DiskErrorException(newErrorText); + throw new DiskErrorException(newErrorText, diskException); } /** Creates a file on the local FS. Pass size as diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index ef54e7dd2c8ea..76da43b4dad24 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -543,8 +543,9 @@ public void testGetLocalPathForWriteForLessSpace() throws Exception { String dir0 = buildBufferDir(ROOT, 0); String dir1 = buildBufferDir(ROOT, 1); conf.set(CONTEXT, dir0 + "," + dir1); - LambdaTestUtils.intercept(DiskErrorException.class, "as the max capacity" + - " in any directory is", "Expect a DiskErrorException.", () -> + LambdaTestUtils.intercept(DiskErrorException.class, + "as the max capacity in any directory is", + "Expect a DiskErrorException.", () -> dirAllocator.getLocalPathForWrite("p1/x", Long.MAX_VALUE-1, conf)); }