MAPREDUCE-7453. Container logs are missing when yarn.app.container.log.filesize is set to default value 0.

Author: zhengchenyu

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java

@@ -64,7 +64,7 @@ public void testCommandLine() throws Exception {
       "  -Xmx820m -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
       " -Dlog4j.configuration=container-log4j.properties" +
       " -Dyarn.app.container.log.dir=<LOG_DIR>" +
-      " -Dyarn.app.container.log.filesize=0" +
+      " -Dyarn.app.container.log.filesize=10240" +
       " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
       " org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
       " 54321" +
@@ -127,7 +127,7 @@ private void testReduceCommandLine(Configuration conf)
             "  -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
             " -Dlog4j.configuration=container-log4j.properties" +
             " -Dyarn.app.container.log.dir=<LOG_DIR>" +
-            " -Dyarn.app.container.log.filesize=0" +
+            " -Dyarn.app.container.log.filesize=10240" +
             " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
             " -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName +
             " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" +
@@ -169,7 +169,7 @@ public void testCommandLineWithLog4JConifg() throws Exception {
       "  -Xmx820m <ADD_OPENS> -Djava.io.tmpdir=" + MRApps.crossPlatformify("PWD") + "/tmp" +
       " -Dlog4j.configuration=" + testLogPropertieFile +
       " -Dyarn.app.container.log.dir=<LOG_DIR>" +
-      " -Dyarn.app.container.log.filesize=0" +
+      " -Dyarn.app.container.log.filesize=10240" +
       " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" +
       " org.apache.hadoop.mapred.YarnChild 127.0.0.1" +
       " 54321" +

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java

@@ -466,7 +466,8 @@ public static long getTaskLogLength(JobConf conf) {
   }
 
   public static long getTaskLogLimitBytes(Configuration conf) {
-    return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024;
+    return conf.getLong(JobContext.TASK_USERLOG_LIMIT, JobContext.DEFAULT_TASK_USERLOG_LIMIT) *
+        1024;
   }
 
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -410,6 +410,8 @@ public interface MRJobConfig {
 
   public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb";
 
+  public static final int DEFAULT_TASK_USERLOG_LIMIT = 10240;
+
   public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent";
 
   public static final String MAP_INPUT_FILE = "mapreduce.map.input.file";
@@ -758,11 +760,11 @@ public interface MRJobConfig {
 
   public static final String MR_AM_LOG_KB =
       MR_AM_PREFIX + "container.log.limit.kb";
-  public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll
+  public static final int DEFAULT_MR_AM_LOG_KB = 10240;
 
   public static final String MR_AM_LOG_BACKUPS =
       MR_AM_PREFIX + "container.log.backups";
-  public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0;
+  public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; // don't roll
 
   /**The number of splits when reporting progress in MR*/
   public static final String MR_AM_NUM_PROGRESS_SPLITS = 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -823,16 +823,15 @@
 
 <property>
   <name>mapreduce.task.userlog.limit.kb</name>
-  <value>0</value>
-  <description>The maximum size of user-logs of each task in KB. 0 disables the cap.
+  <value>10240</value>
+  <description>The maximum size of user-logs of each task in KB.
   </description>
 </property>
 
 <property>
   <name>yarn.app.mapreduce.am.container.log.limit.kb</name>
-  <value>0</value>
+  <value>10240</value>
   <description>The maximum size of the MRAppMaster attempt container logs in KB.
-    0 disables the cap.
   </description>
 </property>
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java

@@ -323,7 +323,7 @@ public void testStartLocalizer() throws IOException {
           String.format("-Dyarn.app.container.log.dir=%s/application_0/12345",
           mockExec.getConf().get(YarnConfiguration.NM_LOG_DIRS)));
       assertThat(result.get(15)).isEqualTo(
-          "-Dyarn.app.container.log.filesize=0");
+          "-Dyarn.app.container.log.filesize=10240");
       assertThat(result.get(16)).isEqualTo("-Dhadoop.root.logger=INFO,CLA");
       assertThat(result.get(17)).isEqualTo(
           "-Dhadoop.root.logfile=container-localizer-syslog");

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -945,7 +945,7 @@ private void verifyTailErrorLogOnContainerExit(Configuration conf,
     invalidCommand.add("-Djava.io.tmpdir=$PWD/tmp");
     invalidCommand.add("-Dlog4j.configuration=container-log4j.properties");
     invalidCommand.add("-Dyarn.app.container.log.dir=" + containerLogDir);
-    invalidCommand.add("-Dyarn.app.container.log.filesize=0");
+    invalidCommand.add("-Dyarn.app.container.log.filesize=10240");
     invalidCommand.add("-Dhadoop.root.logger=INFO,CLA");
     invalidCommand.add("-Dhadoop.root.logfile=syslog");
     invalidCommand.add("-Xmx1024m");