From 1b8bbd9c8953d5da9338c1023f93f2898a2e8482 Mon Sep 17 00:00:00 2001 From: huahuiyang Date: Tue, 25 Aug 2015 14:11:13 +0800 Subject: [PATCH 1/2] Update Shell.java avoid NodeManager out of memory, in case running a command which has verbose error log continuously --- .../src/main/java/org/apache/hadoop/util/Shell.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index e4269558c3069..d1aa6d600913f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -421,7 +421,7 @@ private static boolean isSetsidSupported() { /** Token separator regex used to parse Shell tool outputs */ public static final String TOKEN_SEPARATOR_REGEX = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]"; - + private static final int ERROR_MSG_MAX_LENGTH = 1024 * 1024; private long interval; // refresh interval in msec private long lastTime; // last time the command was performed final private boolean redirectErrorStream; // merge stdout and stderr @@ -528,6 +528,8 @@ public void run() { while((line != null) && !isInterrupted()) { errMsg.append(line); errMsg.append(System.getProperty("line.separator")); + errMsg.delete(0, (errMsg.length() - ERROR_MSG_MAX_LENGTH < 0) ? 0 + : (errMsg.length() - ERROR_MSG_MAX_LENGTH)); line = errReader.readLine(); } } catch(IOException ioe) { From db9c21c47f3c7c211f776ffe961301ce86b28b13 Mon Sep 17 00:00:00 2001 From: huahuiyang Date: Tue, 25 Aug 2015 14:15:17 +0800 Subject: [PATCH 2/2] Update Shell.java avoid NodeManager out of memory, in case running a command which has verbose error log continuously --- .../src/main/java/org/apache/hadoop/util/Shell.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index d1aa6d600913f..3291c28b88ca0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -422,6 +422,7 @@ private static boolean isSetsidSupported() { public static final String TOKEN_SEPARATOR_REGEX = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]"; private static final int ERROR_MSG_MAX_LENGTH = 1024 * 1024; + private long interval; // refresh interval in msec private long lastTime; // last time the command was performed final private boolean redirectErrorStream; // merge stdout and stderr