Skip to content

Commit 194e80a

Browse files
zhengchenyuzhengchenyu
committed
MAPREDUCE-7453. Revert HADOOP-18649. (apache#6102). Contributed by zhengchenyu.
In container-log4j.properties, log4j.appender.{APPENDER}.MaxFileSize is set to ${yarn.app.container.log.filesize}, but yarn.app.container.log.filesize is 0 in default. So log is missing. This log is always rolling and only show the latest log.
1 parent e7545e1 commit 194e80a

File tree

6 files changed

+286
-24
lines changed

6 files changed

+286
-24
lines changed

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
6161
import org.apache.hadoop.util.ApplicationClassLoader;
6262
import org.apache.hadoop.util.StringUtils;
63+
import org.apache.hadoop.yarn.ContainerLogAppender;
64+
import org.apache.hadoop.yarn.ContainerRollingLogAppender;
6365
import org.apache.hadoop.yarn.api.ApplicationConstants;
6466
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
6567
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -586,7 +588,8 @@ public static String getChildLogLevel(Configuration conf, boolean isMap) {
586588

587589
/**
588590
* Add the JVM system properties necessary to configure
589-
* {@link org.apache.log4j.RollingFileAppender}.
591+
* {@link ContainerLogAppender} or
592+
* {@link ContainerRollingLogAppender}.
590593
*
591594
* @param task for map/reduce, or null for app master
592595
* @param vargs the argument list to append to

‎hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -840,17 +840,24 @@
840840
<name>yarn.app.mapreduce.task.container.log.backups</name>
841841
<value>0</value>
842842
<description>Number of backup files for task logs when using
843-
RollingFileAppender (RFA). See
844-
org.apache.log4j.RollingFileAppender.maxBackupIndex.
843+
ContainerRollingLogAppender (CRLA). See
844+
org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
845+
ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
846+
is enabled for tasks when both mapreduce.task.userlog.limit.kb and
847+
yarn.app.mapreduce.task.container.log.backups are greater than zero.
845848
</description>
846849
</property>
847850

848851
<property>
849852
<name>yarn.app.mapreduce.am.container.log.backups</name>
850853
<value>0</value>
851854
<description>Number of backup files for the ApplicationMaster logs when using
852-
RollingFileAppender (RFA). See
853-
org.apache.log4j.RollingFileAppender.maxBackupIndex.
855+
ContainerRollingLogAppender (CRLA). See
856+
org.apache.log4j.RollingFileAppender.maxBackupIndex. By default,
857+
ContainerLogAppender (CLA) is used, and container logs are not rolled. CRLA
858+
is enabled for the ApplicationMaster when both
859+
yarn.app.mapreduce.am.container.log.limit.kb and
860+
yarn.app.mapreduce.am.container.log.backups are greater than zero.
854861
</description>
855862
</property>
856863

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.yarn;
20+
21+
import java.io.File;
22+
import java.io.Flushable;
23+
import java.util.ArrayDeque;
24+
import java.util.Deque;
25+
26+
import org.apache.hadoop.classification.InterfaceAudience.Public;
27+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
28+
import org.apache.log4j.FileAppender;
29+
import org.apache.log4j.spi.LoggingEvent;
30+
31+
/**
32+
* A simple log4j-appender for container's logs.
33+
*/
34+
@Public
35+
@Unstable
36+
public class ContainerLogAppender extends FileAppender
37+
implements Flushable {
38+
39+
private String containerLogDir;
40+
private String containerLogFile;
41+
private int maxEvents;
42+
private Deque<LoggingEvent> eventBuffer;
43+
private boolean closed = false;
44+
45+
@Override
46+
public synchronized void activateOptions() {
47+
if (maxEvents > 0) {
48+
this.eventBuffer = new ArrayDeque<>();
49+
}
50+
setFile(new File(this.containerLogDir, containerLogFile).toString());
51+
setAppend(true);
52+
super.activateOptions();
53+
}
54+
55+
@Override
56+
public synchronized void append(LoggingEvent event) {
57+
if (closed) {
58+
return;
59+
}
60+
if (eventBuffer != null) {
61+
if (eventBuffer.size() == maxEvents) {
62+
eventBuffer.removeFirst();
63+
}
64+
eventBuffer.addLast(event);
65+
} else {
66+
super.append(event);
67+
}
68+
}
69+
70+
@Override
71+
public void flush() {
72+
if (qw != null) {
73+
qw.flush();
74+
}
75+
}
76+
77+
@Override
78+
public synchronized void close() {
79+
if (!closed) {
80+
closed = true;
81+
if (eventBuffer != null) {
82+
for (LoggingEvent event : eventBuffer) {
83+
super.append(event);
84+
}
85+
// let garbage collection do its work
86+
eventBuffer = null;
87+
}
88+
super.close();
89+
}
90+
}
91+
92+
/**
93+
* Getter/Setter methods for log4j.
94+
*
95+
* @return containerLogDir.
96+
*/
97+
public String getContainerLogDir() {
98+
return this.containerLogDir;
99+
}
100+
101+
public void setContainerLogDir(String containerLogDir) {
102+
this.containerLogDir = containerLogDir;
103+
}
104+
105+
public String getContainerLogFile() {
106+
return containerLogFile;
107+
}
108+
109+
public void setContainerLogFile(String containerLogFile) {
110+
this.containerLogFile = containerLogFile;
111+
}
112+
113+
private static final long EVENT_SIZE = 100;
114+
115+
public long getTotalLogFileSize() {
116+
return maxEvents * EVENT_SIZE;
117+
}
118+
119+
/**
120+
* Setter so that log4j can configure it from the
121+
* configuration(log4j.properties).
122+
*
123+
* @param logSize log size.
124+
*/
125+
public void setTotalLogFileSize(long logSize) {
126+
maxEvents = (int)(logSize / EVENT_SIZE);
127+
}
128+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerRollingLogAppender.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.yarn;
20+
21+
import org.apache.hadoop.classification.InterfaceAudience.Public;
22+
import org.apache.hadoop.classification.InterfaceStability.Unstable;
23+
import org.apache.log4j.RollingFileAppender;
24+
25+
import java.io.File;
26+
import java.io.Flushable;
27+
28+
/**
29+
* A simple log4j-appender for container's logs.
30+
*
31+
*/
32+
@Public
33+
@Unstable
34+
public class ContainerRollingLogAppender extends RollingFileAppender implements Flushable {
35+
private String containerLogDir;
36+
private String containerLogFile;
37+
38+
@Override
39+
public void activateOptions() {
40+
synchronized (this) {
41+
setFile(new File(this.containerLogDir, containerLogFile).toString());
42+
setAppend(true);
43+
super.activateOptions();
44+
}
45+
}
46+
47+
@Override
48+
public void flush() {
49+
if (qw != null) {
50+
qw.flush();
51+
}
52+
}
53+
54+
/**
55+
* Getter/Setter methods for log4j.
56+
*
57+
* @return containerLogDir.
58+
*/
59+
60+
public String getContainerLogDir() {
61+
return this.containerLogDir;
62+
}
63+
64+
public void setContainerLogDir(String containerLogDir) {
65+
this.containerLogDir = containerLogDir;
66+
}
67+
68+
public String getContainerLogFile() {
69+
return containerLogFile;
70+
}
71+
72+
public void setContainerLogFile(String containerLogFile) {
73+
this.containerLogFile = containerLogFile;
74+
}
75+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.yarn;
20+
21+
import org.junit.jupiter.api.Test;
22+
23+
import org.apache.log4j.Logger;
24+
import org.apache.log4j.PatternLayout;
25+
26+
public class TestContainerLogAppender {
27+
28+
@Test
29+
void testAppendInClose() throws Exception {
30+
final ContainerLogAppender claAppender = new ContainerLogAppender();
31+
claAppender.setName("testCLA");
32+
claAppender.setLayout(new PatternLayout("%-5p [%t]: %m%n"));
33+
claAppender.setContainerLogDir("target/testAppendInClose/logDir");
34+
claAppender.setContainerLogFile("syslog");
35+
claAppender.setTotalLogFileSize(1000);
36+
claAppender.activateOptions();
37+
final Logger claLog = Logger.getLogger("testAppendInClose-catergory");
38+
claLog.setAdditivity(false);
39+
claLog.addAppender(claAppender);
40+
claLog.info(new Object() {
41+
public String toString() {
42+
claLog.info("message1");
43+
return "return message1";
44+
}
45+
});
46+
claAppender.close();
47+
}
48+
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,35 +26,36 @@ log4j.threshold=ALL
2626
#
2727

2828
#Default values
29-
yarn.app.container.log.filesize=100MB
30-
yarn.app.container.log.backups=1
31-
yarn.app.mapreduce.shuffle.log.backups=1
29+
yarn.app.container.log.dir=null
30+
yarn.app.container.log.filesize=100
3231

33-
log4j.appender.CLA=org.apache.log4j.RollingFileAppender
34-
log4j.appender.CLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile}
35-
log4j.appender.CLA.MaxFileSize=${yarn.app.container.log.filesize}
36-
log4j.appender.CLA.MaxBackupIndex=${yarn.app.container.log.backups}
32+
log4j.appender.CLA=org.apache.hadoop.yarn.ContainerLogAppender
33+
log4j.appender.CLA.containerLogDir=${yarn.app.container.log.dir}
34+
log4j.appender.CLA.containerLogFile=${hadoop.root.logfile}
35+
log4j.appender.CLA.totalLogFileSize=${yarn.app.container.log.filesize}
3736
log4j.appender.CLA.layout=org.apache.log4j.PatternLayout
3837
log4j.appender.CLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n
3938

40-
log4j.appender.CRLA=org.apache.log4j.RollingFileAppender
41-
log4j.appender.CRLA.File=${yarn.app.container.log.dir}/${hadoop.root.logfile}
42-
log4j.appender.CRLA.MaxFileSize=${yarn.app.container.log.filesize}
43-
log4j.appender.CRLA.MaxBackupIndex=${yarn.app.container.log.backups}
39+
log4j.appender.CRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender
40+
log4j.appender.CRLA.containerLogDir=${yarn.app.container.log.dir}
41+
log4j.appender.CRLA.containerLogFile=${hadoop.root.logfile}
42+
log4j.appender.CRLA.maximumFileSize=${yarn.app.container.log.filesize}
43+
log4j.appender.CRLA.maxBackupIndex=${yarn.app.container.log.backups}
4444
log4j.appender.CRLA.layout=org.apache.log4j.PatternLayout
4545
log4j.appender.CRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n
4646

47-
log4j.appender.shuffleCLA=org.apache.log4j.RollingFileAppender
48-
log4j.appender.shuffleCLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile}
49-
log4j.appender.shuffleCLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize}
50-
log4j.appender.shuffleCLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups}
47+
log4j.appender.shuffleCLA=org.apache.hadoop.yarn.ContainerLogAppender
48+
log4j.appender.shuffleCLA.containerLogDir=${yarn.app.container.log.dir}
49+
log4j.appender.shuffleCLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile}
50+
log4j.appender.shuffleCLA.totalLogFileSize=${yarn.app.mapreduce.shuffle.log.filesize}
5151
log4j.appender.shuffleCLA.layout=org.apache.log4j.PatternLayout
5252
log4j.appender.shuffleCLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n
5353

54-
log4j.appender.shuffleCRLA=org.apache.log4j.RollingFileAppender
55-
log4j.appender.shuffleCRLA.File=${yarn.app.container.log.dir}/${yarn.app.mapreduce.shuffle.logfile}
56-
log4j.appender.shuffleCRLA.MaxFileSize=${yarn.app.mapreduce.shuffle.log.filesize}
57-
log4j.appender.shuffleCRLA.MaxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups}
54+
log4j.appender.shuffleCRLA=org.apache.hadoop.yarn.ContainerRollingLogAppender
55+
log4j.appender.shuffleCRLA.containerLogDir=${yarn.app.container.log.dir}
56+
log4j.appender.shuffleCRLA.containerLogFile=${yarn.app.mapreduce.shuffle.logfile}
57+
log4j.appender.shuffleCRLA.maximumFileSize=${yarn.app.mapreduce.shuffle.log.filesize}
58+
log4j.appender.shuffleCRLA.maxBackupIndex=${yarn.app.mapreduce.shuffle.log.backups}
5859
log4j.appender.shuffleCRLA.layout=org.apache.log4j.PatternLayout
5960
log4j.appender.shuffleCRLA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c: %m%n
6061

0 commit comments

Comments
 (0)