From 748fa05119f192c6a92794f4b4aeea51aae62520 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 15 Sep 2022 19:51:01 +0800 Subject: [PATCH 1/4] YARN-11305. Fix TestLogAggregationService#testLocalFileDeletionAfterUpload Failed After YARN-11241(#4703). --- .../logaggregation/TestLogAggregationService.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 8185f5019c798..840b0ada56f23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -202,11 +202,11 @@ public void tearDown() throws IOException, InterruptedException { } private void verifyLocalFileDeletion( - LogAggregationService logAggregationService) throws Exception { + LogAggregationService logAggregationService,Long clusterTimeStamp) throws Exception { logAggregationService.init(this.conf); logAggregationService.start(); - ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1); + ApplicationId application1 = BuilderUtils.newApplicationId(clusterTimeStamp, 1); // AppLogDir should be created File app1LogDir = @@ -252,7 +252,7 @@ private void verifyLocalFileDeletion( count = 0; while ((f.exists()) && (count < maxAttempts)) { count++; - Thread.sleep(100); + Thread.sleep(1000); } Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); } @@ -310,7 +310,7 @@ public void testLocalFileDeletionAfterUpload() throws Exception { LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); - verifyLocalFileDeletion(logAggregationService); + verifyLocalFileDeletion(logAggregationService, 1234L); } @Test @@ -324,7 +324,7 @@ public void testLocalFileRemainsAfterUploadOnCleanupDisable() throws Exception { this.remoteRootLogDir.getAbsolutePath()); LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); - verifyLocalFileDeletion(logAggregationService); + verifyLocalFileDeletion(logAggregationService, 4321L); } @Test @@ -344,7 +344,7 @@ public void testLocalFileDeletionOnDiskFull() throws Exception { LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, dirsHandler)); - verifyLocalFileDeletion(logAggregationService); + verifyLocalFileDeletion(logAggregationService, 5678L); } /* Test to verify fix for YARN-3793 */ From 1c89c3cbc5900bd46fad129129debb8e2c0f43d9 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 15 Sep 2022 06:29:21 -0700 Subject: [PATCH 2/4] YARN-11305. Fix CheckStyle. --- .../logaggregation/TestLogAggregationService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 840b0ada56f23..e5cf40a411532 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -202,7 +202,7 @@ public void tearDown() throws IOException, InterruptedException { } private void verifyLocalFileDeletion( - LogAggregationService logAggregationService,Long clusterTimeStamp) throws Exception { + LogAggregationService logAggregationService, Long clusterTimeStamp) throws Exception { logAggregationService.init(this.conf); logAggregationService.start(); From d553d26df3f0a4c1fdd3d6261d45c5a09f044073 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 16 Sep 2022 17:13:30 +0800 Subject: [PATCH 3/4] YARN-11305. Fix CheckStyle. --- .../logaggregation/TestLogAggregationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index e5cf40a411532..ca858ffbc8d3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -56,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; From b62f0abdd6bd86bd7f75958250258b50c9ed5297 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Fri, 16 Sep 2022 17:14:54 +0800 Subject: [PATCH 4/4] YARN-11305.Fix CheckStyle. --- .../TestLogAggregationService.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index ca858ffbc8d3f..4ec8f462f5117 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -203,10 +203,12 @@ public void tearDown() throws IOException, InterruptedException { } private void verifyLocalFileDeletion( - LogAggregationService logAggregationService, Long clusterTimeStamp) throws Exception { + LogAggregationService logAggregationService) throws Exception { logAggregationService.init(this.conf); logAggregationService.start(); + Random random = new Random(System.currentTimeMillis()); + long clusterTimeStamp = random.nextLong(); ApplicationId application1 = BuilderUtils.newApplicationId(clusterTimeStamp, 1); // AppLogDir should be created @@ -246,15 +248,9 @@ private void verifyLocalFileDeletion( String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); - int count = 0; - int maxAttempts = 50; for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { File f = new File(containerLogDir, fileType); - count = 0; - while ((f.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(1000); - } + GenericTestUtils.waitFor(() -> !f.exists(), 1000, 1000 * 50); Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); } Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", @@ -311,7 +307,7 @@ public void testLocalFileDeletionAfterUpload() throws Exception { LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); - verifyLocalFileDeletion(logAggregationService, 1234L); + verifyLocalFileDeletion(logAggregationService); } @Test @@ -325,7 +321,7 @@ public void testLocalFileRemainsAfterUploadOnCleanupDisable() throws Exception { this.remoteRootLogDir.getAbsolutePath()); LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); - verifyLocalFileDeletion(logAggregationService, 4321L); + verifyLocalFileDeletion(logAggregationService); } @Test @@ -345,7 +341,7 @@ public void testLocalFileDeletionOnDiskFull() throws Exception { LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, dirsHandler)); - verifyLocalFileDeletion(logAggregationService, 5678L); + verifyLocalFileDeletion(logAggregationService); } /* Test to verify fix for YARN-3793 */