From 05a0e93253342fc0a9a578220abcf5aec01cc173 Mon Sep 17 00:00:00 2001 From: Ramesh Thangarajan Date: Tue, 17 Dec 2019 09:48:08 -0800 Subject: [PATCH 1/3] HADOOP-16769 Log details of requested size and available capacity when file creation is not successuful --- .../main/java/org/apache/hadoop/fs/LocalDirAllocator.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java index 5f266a7b82555..cf8920b03aaa0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java @@ -393,6 +393,7 @@ public Path getLocalPathForWrite(String pathStr, long size, Context ctx = confChanged(conf); int numDirs = ctx.localDirs.length; int numDirsSearched = 0; + long maxCapacity = 0; //remove the leading slash from the path (to make sure that the uri //resolution results in a valid path on the dir being checked) if (pathStr.startsWith("/")) { @@ -441,6 +442,9 @@ public Path getLocalPathForWrite(String pathStr, long size, int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc); while (numDirsSearched < numDirs) { long capacity = ctx.dirDF[dirNum].getAvailable(); + if (capacity > maxCapacity) { + maxCapacity = capacity; + } if (capacity > size) { returnPath = createPath(ctx.localDirs[dirNum], pathStr, checkWrite); @@ -460,7 +464,8 @@ public Path getLocalPathForWrite(String pathStr, long size, //no path found throw new DiskErrorException("Could not find any valid local " + - "directory for " + pathStr); + "directory for " + pathStr + " with requested size " + size + + " as the max capacity in any directory is " + maxCapacity); } /** Creates a file on the local FS. Pass size as From 23d8353dff4d45dbfbbc7fa6d1ff411153524f01 Mon Sep 17 00:00:00 2001 From: Ramesh Thangarajan Date: Wed, 18 Dec 2019 22:09:42 -0800 Subject: [PATCH 2/3] HADOOP-16769 Log details of requested size and available capacity when file creation is not successuful --- .../hadoop/fs/TestLocalDirAllocator.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index acda898ea1342..6e239bf9ffe4f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -532,4 +532,23 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception { } } + /** + * Test to check the LocalDirAllocation for the invalid path HADOOP-8437 + * + * @throws Exception + */ + @Test(timeout = 30000) + public void testGetLocalPathForWriteForLessSpace() throws Exception { + String dir0 = buildBufferDir(ROOT, 0); + String dir1 = buildBufferDir(ROOT, 1); + conf.set(CONTEXT, dir0 + "," + dir1); + try { + dirAllocator.getLocalPathForWrite("p1/x", 3_000_000_000_000L, conf); + fail("not throwing the exception"); + } catch (IOException e) { + assertTrue(e.getMessage().matches("Could not find any valid local directory for p1/x with requested " + + "size 3000000000000 as the max capacity in any directory is [0-9]*")); + } + } + } From 6ae80b032451d1a95f3b9389adf0d260102bfd24 Mon Sep 17 00:00:00 2001 From: Ramesh Thangarajan Date: Wed, 18 Dec 2019 22:11:21 -0800 Subject: [PATCH 3/3] HADOOP-16769 Log details of requested size and available capacity when file creation is not successuful --- .../test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index 6e239bf9ffe4f..ecc5a45000e3a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -533,7 +533,7 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception { } /** - * Test to check the LocalDirAllocation for the invalid path HADOOP-8437 + * Test to check the LocalDirAllocation for the less space HADOOP-16769 * * @throws Exception */