1+ /*
2+ * Licensed under the Apache License, Version 2.0 (the "License");
3+ * you may not use this file except in compliance with the License.
4+ * You may obtain a copy of the License at
5+ *
6+ * http://www.apache.org/licenses/LICENSE-2.0
7+ *
8+ * Unless required by applicable law or agreed to in writing, software
9+ * distributed under the License is distributed on an "AS IS" BASIS,
10+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+ * See the License for the specific language governing permissions and
12+ * limitations under the License.
13+ */
14+ package org .apache .spark .log4j ;
15+ import java .io .IOException ;
16+ import java .text .SimpleDateFormat ;
17+ import java .util .Date ;
18+ import java .util .UUID ;
19+
20+ import org .apache .commons .lang3 .StringUtils ;
21+ import org .apache .log4j .FileAppender ;
22+ import org .apache .log4j .Layout ;
23+ import org .apache .log4j .helpers .LogLog ;
24+ import org .apache .spark .SparkEnv ;
25+
26+ import com .google .common .annotations .VisibleForTesting ;
27+
28+ public class LocalFileAppender extends FileAppender {
29+ private String metadataIdentifier ;
30+ private String category ;
31+ private String identifier ;
32+ private String jobName ;
33+ private String project ;
34+ private String executorId ;
35+ private String mountDir ;
36+
37+ private SimpleDateFormat dateFormat = new SimpleDateFormat ("yyyy-MM-dd" );
38+
39+ public LocalFileAppender () {
40+ super ();
41+ }
42+
43+ public LocalFileAppender (Layout layout , String filename , boolean append ) throws IOException {
44+ super (layout , filename , append );
45+ }
46+
47+ public LocalFileAppender (Layout layout , String filename ) throws IOException {
48+ super (layout , filename );
49+ }
50+
51+ @ Override
52+ public void activateOptions () {
53+ LogLog .warn (String .format ("%s starting ..." , name ));
54+ this .fileName = getOutPutDir ();
55+ LogLog .warn (String .format ("Output path is %s" , this .fileName ));
56+ LogLog .warn ("metadataIdentifier -> " + getMetadataIdentifier ());
57+ LogLog .warn ("category -> " + getCategory ());
58+ LogLog .warn ("identifier -> " + getIdentifier ());
59+ LogLog .warn ("mountDir -> " + getMountDir ());
60+
61+ if (null != getProject ()) {
62+ LogLog .warn ("project -> " + getProject ());
63+ }
64+
65+ if (null != getJobName ()) {
66+ LogLog .warn ("jobName -> " + getJobName ());
67+ }
68+ super .activateOptions ();
69+ }
70+
71+ @ VisibleForTesting
72+ String getOutPutDir () {
73+ String rollingDir = dateFormat .format (new Date ());
74+ if (StringUtils .isBlank (executorId )) {
75+ executorId = SparkEnv .get () != null ? SparkEnv .get ().executorId () : UUID .randomUUID ().toString ();
76+ LogLog .warn ("executorId set to " + executorId );
77+ }
78+
79+ if ("job" .equals (getCategory ())) {
80+ return getRootPathName () + "/" + rollingDir +
81+ "/" + getIdentifier () + "/" + getJobName () + "/" + "executor-"
82+ + executorId + ".log" ;
83+ }
84+ return getRootPathName () + "/" + rollingDir
85+ + "/" + getIdentifier () + "/" + "executor-" + executorId + ".log" ;
86+ }
87+
88+ String getRootPathName () {
89+ if (!mountDir .endsWith ("/" )) {
90+ mountDir = mountDir + "/" ;
91+ }
92+ if ("job" .equals (getCategory ())) {
93+ return mountDir + getProject () + "/spark_logs" ;
94+ } else if ("sparder" .equals (getCategory ())) {
95+ return mountDir + "_sparder_logs" ;
96+ } else {
97+ throw new IllegalArgumentException ("illegal category: " + getCategory ());
98+ }
99+ }
100+
101+
102+ public String getMetadataIdentifier () {
103+ return metadataIdentifier ;
104+ }
105+
106+ public void setMetadataIdentifier (String metadataIdentifier ) {
107+ this .metadataIdentifier = metadataIdentifier ;
108+ }
109+
110+ public String getCategory () {
111+ return category ;
112+ }
113+
114+ public void setCategory (String category ) {
115+ this .category = category ;
116+ }
117+
118+ public String getIdentifier () {
119+ return identifier ;
120+ }
121+
122+ public void setIdentifier (String identifier ) {
123+ this .identifier = identifier ;
124+ }
125+
126+ public String getJobName () {
127+ return jobName ;
128+ }
129+
130+ public void setJobName (String jobName ) {
131+ this .jobName = jobName ;
132+ }
133+
134+ public String getProject () {
135+ return project ;
136+ }
137+
138+ public void setProject (String project ) {
139+ this .project = project ;
140+ }
141+
142+ public String getExecutorId () {
143+ return executorId ;
144+ }
145+
146+ public void setExecutorId (String executorId ) {
147+ this .executorId = executorId ;
148+ }
149+
150+ public String getMountDir () {
151+ return mountDir ;
152+ }
153+
154+ public void setMountDir (String mountDir ) {
155+ this .mountDir = mountDir ;
156+ }
157+
158+ }
0 commit comments