From 2a1f223b1f7a3d423ff2cb2400eab7a1cf17c474 Mon Sep 17 00:00:00 2001 From: Kiran Kumar Maturi Date: Tue, 27 Aug 2024 11:30:28 +0530 Subject: [PATCH 1/3] HBASE-28801 WALs are not cleaned even after all entries are flushed --- .../apache/hadoop/hbase/regionserver/wal/FSHLog.java | 6 +++--- .../hadoop/hbase/regionserver/wal/TestFSHLog.java | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 88e59c6d4b16..f15000a53061 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -390,10 +390,10 @@ protected void doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter) th try { closeWriter(this.writer, oldPath, true); } finally { + //closing this with unflushed entries as there is no other chance we can set close to true + // during clean up we check for unflushed entries + markClosedAndClean(oldPath); inflightWALClosures.remove(oldPath.getName()); - if (!isUnflushedEntries()) { - markClosedAndClean(oldPath); - } } } else { Writer localWriter = this.writer; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java index ec2231b3f98b..ce237a31264c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java @@ -324,7 +324,7 @@ public void run() { /** * Test for jira https://issues.apache.org/jira/browse/HBASE-28665 */ - public void testWALClosureFailureAndCleanup() throws IOException { + public void testWALClosureFailureAndCleanup() throws IOException, InterruptedException { class FailingWriter implements WALProvider.Writer { @Override @@ -375,12 +375,15 @@ public void close() throws IOException { region.put(new Put(b).addColumn(b, b, b)); log.rollWriter(); } - assertEquals(2, log.getClosedErrorCount()); region.put(new Put(b).addColumn(b, b, b)); region.put(new Put(b).addColumn(b, b, b)); - region.flush(true); + log.highestUnsyncedTxid = log.highestSyncedTxid.get() + 100; + assertTrue("WAL has unflushed entries ", log.getUnflushedEntriesCount() > 0); log.rollWriter(); - assertEquals("WAL Files not cleaned ", 0, log.walFile2Props.size()); + assertEquals("WAL Files not cleaned ", 3, log.walFile2Props.size()); + region.flush(true); + log.markClosedAndClean(log.walFile2Props.firstKey()); + assertEquals("WAL Files not cleaned ", 1, log.walFile2Props.size()); region.close(); } } From 69cdb543791d8243a79f78359ae95f0c9b61f31e Mon Sep 17 00:00:00 2001 From: Kiran Kumar Maturi Date: Tue, 27 Aug 2024 15:44:07 +0530 Subject: [PATCH 2/3] spotless fixes --- .../java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index f15000a53061..69a6b1ecf94e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -390,7 +390,7 @@ protected void doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter) th try { closeWriter(this.writer, oldPath, true); } finally { - //closing this with unflushed entries as there is no other chance we can set close to true + // closing this as there is no other chance we can set close to true // during clean up we check for unflushed entries markClosedAndClean(oldPath); inflightWALClosures.remove(oldPath.getName()); From 17ce08cc173e2f89e339708847bb29801bd13009 Mon Sep 17 00:00:00 2001 From: Kiran Kumar Maturi Date: Mon, 9 Sep 2024 11:18:49 +0530 Subject: [PATCH 3/3] added logging for files with unflushed entries --- .../org/apache/hadoop/hbase/regionserver/wal/FSHLog.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 69a6b1ecf94e..bd464f236a36 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -390,10 +390,12 @@ protected void doReplaceWriter(Path oldPath, Path newPath, Writer nextWriter) th try { closeWriter(this.writer, oldPath, true); } finally { - // closing this as there is no other chance we can set close to true - // during clean up we check for unflushed entries - markClosedAndClean(oldPath); inflightWALClosures.remove(oldPath.getName()); + if (!isUnflushedEntries()) { + markClosedAndClean(oldPath); + } else { + LOG.debug("WAL has unflushed entries path: " + oldPath); + } } } else { Writer localWriter = this.writer;