From fc83ec11c21de991aac219c3e0cfe995408f804d Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Thu, 4 Aug 2022 19:16:16 +0100 Subject: [PATCH 1/6] YARN-11241. Add uncleaning option for local app log file with log-aggregation enabled --- .../hadoop/yarn/conf/YarnConfiguration.java | 7 ++ .../logaggregation/AppLogAggregatorImpl.java | 44 ++++++---- .../TestLogAggregationService.java | 86 ++++++++++++++----- 3 files changed, 99 insertions(+), 38 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d42562cf6140a..d82f39a9e1e2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1548,6 +1548,13 @@ public static boolean isAclEnabled(Configuration conf) { public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS = 10 * 60 * 1000; + /** + * Whether to clean up after nodemanager logs when log aggregation is enabled + */ + public static final String LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = + YARN_PREFIX + "log-aggregation.enable-local-cleanup"; + public static final boolean DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = true; + /** * Number of seconds to retain logs on the NodeManager. Only applicable if Log * aggregation is disabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 96e08c5ff38ee..b41cacdca0f8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -86,6 +86,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final Dispatcher dispatcher; private final ApplicationId appId; private final String applicationId; + private final boolean enableLocalCleanup; private boolean logAggregationDisabled = false; private final Configuration conf; private final DeletionService delService; @@ -172,6 +173,9 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.logAggregationContext = logAggregationContext; this.context = context; this.nodeId = nodeId; + this.enableLocalCleanup = + conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; this.logFileSizeThreshold = @@ -337,26 +341,26 @@ private void uploadLogsForContainers(boolean appFinished) appFinished, finishedContainers.contains(container)); if (uploadedFilePathsInThisCycle.size() > 0) { uploadedLogsInThisCycle = true; - LOG.trace("Uploaded the following files for {}: {}", - container, uploadedFilePathsInThisCycle.toString()); - List uploadedFilePathsInThisCycleList = new ArrayList<>(); - uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); - if (LOG.isDebugEnabled()) { - for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { - try { - long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); - if (fileSize >= logFileSizeThreshold) { - LOG.debug("Log File " + uploadedFilePath - + " size is " + fileSize + " bytes"); + if (enableLocalCleanup) { + LOG.trace("Uploaded the following files for {}: {}", container, + uploadedFilePathsInThisCycle.toString()); + List uploadedFilePathsInThisCycleList = new ArrayList<>(); + uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + try { + long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if (fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + " size is " + fileSize + " bytes"); + } + } catch (Exception e1) { + LOG.error("Failed to get log file size " + e1); } - } catch (Exception e1) { - LOG.error("Failed to get log file size " + e1); } } + deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, + uploadedFilePathsInThisCycleList); } - deletionTask = new FileDeletionTask(delService, - this.userUgi.getShortUserName(), null, - uploadedFilePathsInThisCycleList); } // This container is finished, and all its logs have been uploaded, @@ -472,7 +476,9 @@ public void run() { // do post clean up of log directories on any other exception LOG.error("Error occurred while aggregating the log for the application " + appId, e); - doAppLogAggregationPostCleanUp(); + if (enableLocalCleanup) { + doAppLogAggregationPostCleanUp(); + } } finally { if (!this.appAggregationFinished.get() && !this.aborted.get()) { LOG.warn("Log aggregation did not complete for application " + appId); @@ -516,7 +522,9 @@ private void doAppLogAggregation() throws LogAggregationDFSException { // App is finished, upload the container logs. uploadLogsForContainers(true); - doAppLogAggregationPostCleanUp(); + if (enableLocalCleanup) { + doAppLogAggregationPostCleanUp(); + } } catch (LogAggregationDFSException e) { LOG.error("Error during log aggregation", e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 4cc9ac1f3a7a4..20266c4b43d15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -234,31 +234,63 @@ private void verifyLocalFileDeletion( // ensure filesystems were closed verify(logAggregationService).closeFileSystems( any(UserGroupInformation.class)); - List dirList = new ArrayList<>(); - dirList.add(new Path(app1LogDir.toURI())); - verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher( - delSrvc, user, null, dirList))); - - String containerIdStr = container11.toString(); - File containerLogDir = new File(app1LogDir, containerIdStr); - int count = 0; - int maxAttempts = 50; - for (String fileType : new String[] { "stdout", "stderr", "syslog" }) { - File f = new File(containerLogDir, fileType); + boolean filesShouldBeDeleted = + this.conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + if (filesShouldBeDeleted) { + List dirList = new ArrayList<>(); + dirList.add(new Path(app1LogDir.toURI())); + verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher( + delSrvc, user, null, dirList))); + + String containerIdStr = container11.toString(); + File containerLogDir = new File(app1LogDir, containerIdStr); + int count = 0; + int maxAttempts = 50; + for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { + File f = new File(containerLogDir, fileType); + count = 0; + while ((f.exists()) && (count < maxAttempts)) { + count++; + Thread.sleep(100); + } + Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); + } count = 0; - while ((f.exists()) && (count < maxAttempts)) { + while ((app1LogDir.exists()) && (count < maxAttempts)) { count++; Thread.sleep(100); } - Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); - } - count = 0; - while ((app1LogDir.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); + Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", + app1LogDir.exists()); + } else { + List dirList = new ArrayList<>(); + dirList.add(new Path(app1LogDir.toURI())); + verify(delSrvc, never()).delete(argThat(new FileDeletionMatcher( + delSrvc, user, null, dirList))); + + String containerIdStr = container11.toString(); + File containerLogDir = new File(app1LogDir, containerIdStr); + int count = 0; + int maxAttempts = 50; + for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { + File f = new File(containerLogDir, fileType); + count = 0; + while ((f.exists()) && (count < maxAttempts)) { + count++; + Thread.sleep(100); + } + Assert.assertTrue("File [" + f + "] was not deleted", f.exists()); + } + count = 0; + while ((app1LogDir.exists()) && (count < maxAttempts)) { + count++; + Thread.sleep(100); + } + Assert.assertTrue("Directory [" + app1LogDir + "] was not deleted", + app1LogDir.exists()); } - Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", - app1LogDir.exists()); + delSrvc.stop(); Path logFilePath = logAggregationService .getLogAggregationFileController(conf) @@ -297,6 +329,20 @@ public void testLocalFileDeletionAfterUpload() throws Exception { verifyLocalFileDeletion(logAggregationService); } + @Test + public void testLocalFileRemainsAfterUploadOnCleanupDisable() throws Exception { + this.delSrvc = new DeletionService(createContainerExecutor()); + delSrvc = spy(delSrvc); + this.delSrvc.init(conf); + this.conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, false); + this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + this.remoteRootLogDir.getAbsolutePath()); + LogAggregationService logAggregationService = spy( + new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler)); + verifyLocalFileDeletion(logAggregationService); + } + @Test public void testLocalFileDeletionOnDiskFull() throws Exception { this.delSrvc = new DeletionService(createContainerExecutor()); From dad54d34a12f75a1dff228b4bd9881ca154dc46f Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Fri, 5 Aug 2022 10:10:14 +0100 Subject: [PATCH 2/6] Adding configuration property to yarn-default.xml --- .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d82f39a9e1e2d..c49038daf3bf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1549,7 +1549,7 @@ public static boolean isAclEnabled(Configuration conf) { = 10 * 60 * 1000; /** - * Whether to clean up after nodemanager logs when log aggregation is enabled + * Whether to clean up nodemanager logs when log aggregation is enabled */ public static final String LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = YARN_PREFIX + "log-aggregation.enable-local-cleanup"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 407ef74d3d062..a33e81a0e3ecb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1548,6 +1548,13 @@ 600000 + + Whether to clean up nodemanager logs when log aggregation is enabled + + yarn.log-aggregation.enable-local-cleanup + true + + Time in seconds to retain user logs. Only applicable if log aggregation is disabled From 7c52957074d0aefb5fc3a62bb48fc043c5cbf4cd Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 30 Aug 2022 11:23:10 +0100 Subject: [PATCH 3/6] Addressing comments --- .../src/main/resources/yarn-default.xml | 4 +++- .../logaggregation/AppLogAggregatorImpl.java | 15 +++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a33e81a0e3ecb..7c3bc22f4c615 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1549,7 +1549,9 @@ - Whether to clean up nodemanager logs when log aggregation is enabled + Whether to clean up nodemanager logs when log aggregation is enabled. Setting to + false disables the cleanup nodemanager logging, and it causes disk full in the long run. Users + can set to false for test-only purpose. yarn.log-aggregation.enable-local-cleanup true diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index b41cacdca0f8c..1ba7353a1eef3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -176,6 +176,10 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.enableLocalCleanup = conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP, YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + if (!this.enableLocalCleanup) { + LOG.warn("{} is only for testing and not for any production system ", + YarnConfiguration.LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP); + } this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; this.logFileSizeThreshold = @@ -476,9 +480,7 @@ public void run() { // do post clean up of log directories on any other exception LOG.error("Error occurred while aggregating the log for the application " + appId, e); - if (enableLocalCleanup) { - doAppLogAggregationPostCleanUp(); - } + doAppLogAggregationPostCleanUp(); } finally { if (!this.appAggregationFinished.get() && !this.aborted.get()) { LOG.warn("Log aggregation did not complete for application " + appId); @@ -522,9 +524,7 @@ private void doAppLogAggregation() throws LogAggregationDFSException { // App is finished, upload the container logs. uploadLogsForContainers(true); - if (enableLocalCleanup) { - doAppLogAggregationPostCleanUp(); - } + doAppLogAggregationPostCleanUp(); } catch (LogAggregationDFSException e) { LOG.error("Error during log aggregation", e); } @@ -536,6 +536,9 @@ private void doAppLogAggregation() throws LogAggregationDFSException { } private void doAppLogAggregationPostCleanUp() { + if (!enableLocalCleanup) { + return; + } // Remove the local app-log-dirs List localAppLogDirs = new ArrayList(); for (String rootLogDir : dirsHandler.getLogDirsForCleanup()) { From 04811718931f33ea2af27aebf8c5fc4a79be6256 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Tue, 30 Aug 2022 13:45:52 +0100 Subject: [PATCH 4/6] Removing not required while loops --- .../TestLogAggregationService.java | 26 ++----------------- 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 20266c4b43d15..569719a5bb678 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -245,22 +245,11 @@ private void verifyLocalFileDeletion( String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); - int count = 0; - int maxAttempts = 50; for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { File f = new File(containerLogDir, fileType); - count = 0; - while ((f.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); - } + Thread.sleep(5000); Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); } - count = 0; - while ((app1LogDir.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); - } Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", app1LogDir.exists()); } else { @@ -271,22 +260,11 @@ private void verifyLocalFileDeletion( String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); - int count = 0; - int maxAttempts = 50; for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { File f = new File(containerLogDir, fileType); - count = 0; - while ((f.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); - } + Thread.sleep(5000); Assert.assertTrue("File [" + f + "] was not deleted", f.exists()); } - count = 0; - while ((app1LogDir.exists()) && (count < maxAttempts)) { - count++; - Thread.sleep(100); - } Assert.assertTrue("Directory [" + app1LogDir + "] was not deleted", app1LogDir.exists()); } From 02c16cdd8e142bab58887ec3a4f3d7748398afb2 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Wed, 31 Aug 2022 01:57:20 +0100 Subject: [PATCH 5/6] Addding period in javadoc --- .../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c49038daf3bf3..5db6122fbcbbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1549,7 +1549,7 @@ public static boolean isAclEnabled(Configuration conf) { = 10 * 60 * 1000; /** - * Whether to clean up nodemanager logs when log aggregation is enabled + * Whether to clean up nodemanager logs when log aggregation is enabled. */ public static final String LOG_AGGREGATION_ENABLE_LOCAL_CLEANUP = YARN_PREFIX + "log-aggregation.enable-local-cleanup"; From 33edb8cf277f714ff124fcb869a61a4eeb092b39 Mon Sep 17 00:00:00 2001 From: Ashutosh Gupta Date: Mon, 12 Sep 2022 06:02:49 +0100 Subject: [PATCH 6/6] adressing comments related to sleep --- .../logaggregation/TestLogAggregationService.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 569719a5bb678..8185f5019c798 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -245,9 +245,15 @@ private void verifyLocalFileDeletion( String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); + int count = 0; + int maxAttempts = 50; for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { File f = new File(containerLogDir, fileType); - Thread.sleep(5000); + count = 0; + while ((f.exists()) && (count < maxAttempts)) { + count++; + Thread.sleep(100); + } Assert.assertFalse("File [" + f + "] was not deleted", f.exists()); } Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", @@ -260,9 +266,9 @@ private void verifyLocalFileDeletion( String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); + Thread.sleep(5000); for (String fileType : new String[]{"stdout", "stderr", "syslog"}) { File f = new File(containerLogDir, fileType); - Thread.sleep(5000); Assert.assertTrue("File [" + f + "] was not deleted", f.exists()); } Assert.assertTrue("Directory [" + app1LogDir + "] was not deleted",