diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index 5f266a7b82555..cf8920b03aaa0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -393,6 +393,7 @@ public Path getLocalPathForWrite(String pathStr, long size, Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; + long maxCapacity = 0; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { @@ -441,6 +442,9 @@ public Path getLocalPathForWrite(String pathStr, long size, int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc); while (numDirsSearched < numDirs) { long capacity = ctx.dirDF[dirNum].getAvailable(); + if (capacity > maxCapacity) { + maxCapacity = capacity; + } if (capacity > size) { returnPath = createPath(ctx.localDirs[dirNum], pathStr, checkWrite); @@ -460,7 +464,8 @@ public Path getLocalPathForWrite(String pathStr, long size, //no path found throw new DiskErrorException("Could not find any valid local " + - "directory for " + pathStr); + "directory for " + pathStr + " with requested size " + size + + " as the max capacity in any directory is " + maxCapacity); } /** Creates a file on the local FS. Pass size as diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index acda898ea1342..ecc5a45000e3a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -532,4 +532,23 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception { } } + /** + * Test to check the LocalDirAllocation for the less space HADOOP-16769 + * + * @throws Exception + */ + @Test(timeout = 30000) + public void testGetLocalPathForWriteForLessSpace() throws Exception { + String dir0 = buildBufferDir(ROOT, 0); + String dir1 = buildBufferDir(ROOT, 1); + conf.set(CONTEXT, dir0 + "," + dir1); + try { + dirAllocator.getLocalPathForWrite("p1/x", 3_000_000_000_000L, conf); + fail("not throwing the exception"); + } catch (IOException e) { + assertTrue(e.getMessage().matches("Could not find any valid local directory for p1/x with requested " + + "size 3000000000000 as the max capacity in any directory is [0-9]*")); + } + } + }