Skip to content

Commit ff27192

Browse files
SirOibafvatj
authored andcommitted
Reduce number of event logs flushes (#26)
1 parent e21a008 commit ff27192

File tree

1 file changed

+23
-23
lines changed

1 file changed

+23
-23
lines changed

core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -152,96 +152,96 @@ private[spark] class EventLoggingListener(
152152
}
153153

154154
// log stage completed event
155-
logEvent(event, flushLogger = true)
155+
logEvent(event)
156156
}
157157

158158
override def onJobStart(event: SparkListenerJobStart): Unit = {
159-
logEvent(event.copy(properties = redactProperties(event.properties)), flushLogger = true)
159+
logEvent(event.copy(properties = redactProperties(event.properties)))
160160
}
161161

162162
override def onJobEnd(event: SparkListenerJobEnd): Unit = logEvent(event, flushLogger = true)
163163

164164
override def onBlockManagerAdded(event: SparkListenerBlockManagerAdded): Unit = {
165-
logEvent(event, flushLogger = true)
165+
logEvent(event)
166166
}
167167

168168
override def onBlockManagerRemoved(event: SparkListenerBlockManagerRemoved): Unit = {
169-
logEvent(event, flushLogger = true)
169+
logEvent(event)
170170
}
171171

172172
override def onUnpersistRDD(event: SparkListenerUnpersistRDD): Unit = {
173-
logEvent(event, flushLogger = true)
173+
logEvent(event)
174174
}
175175

176176
override def onApplicationStart(event: SparkListenerApplicationStart): Unit = {
177-
logEvent(event, flushLogger = true)
177+
logEvent(event)
178178
}
179179

180180
override def onApplicationEnd(event: SparkListenerApplicationEnd): Unit = {
181181
logEvent(event, flushLogger = true)
182182
}
183183
override def onExecutorAdded(event: SparkListenerExecutorAdded): Unit = {
184-
logEvent(event, flushLogger = true)
184+
logEvent(event)
185185
}
186186

187187
override def onExecutorRemoved(event: SparkListenerExecutorRemoved): Unit = {
188-
logEvent(event, flushLogger = true)
188+
logEvent(event)
189189
}
190190

191191
override def onExecutorBlacklisted(event: SparkListenerExecutorBlacklisted): Unit = {
192-
logEvent(event, flushLogger = true)
192+
logEvent(event)
193193
}
194194

195195
override def onExecutorExcluded(event: SparkListenerExecutorExcluded): Unit = {
196-
logEvent(event, flushLogger = true)
196+
logEvent(event)
197197
}
198198

199199
override def onExecutorBlacklistedForStage(
200200
event: SparkListenerExecutorBlacklistedForStage): Unit = {
201-
logEvent(event, flushLogger = true)
201+
logEvent(event)
202202
}
203203

204204
override def onExecutorExcludedForStage(
205205
event: SparkListenerExecutorExcludedForStage): Unit = {
206-
logEvent(event, flushLogger = true)
206+
logEvent(event)
207207
}
208208

209209
override def onNodeBlacklistedForStage(event: SparkListenerNodeBlacklistedForStage): Unit = {
210-
logEvent(event, flushLogger = true)
210+
logEvent(event)
211211
}
212212

213213
override def onNodeExcludedForStage(event: SparkListenerNodeExcludedForStage): Unit = {
214-
logEvent(event, flushLogger = true)
214+
logEvent(event)
215215
}
216216

217217
override def onExecutorUnblacklisted(event: SparkListenerExecutorUnblacklisted): Unit = {
218-
logEvent(event, flushLogger = true)
218+
logEvent(event)
219219
}
220220

221221
override def onExecutorUnexcluded(event: SparkListenerExecutorUnexcluded): Unit = {
222-
logEvent(event, flushLogger = true)
222+
logEvent(event)
223223
}
224224

225225

226226
override def onNodeBlacklisted(event: SparkListenerNodeBlacklisted): Unit = {
227-
logEvent(event, flushLogger = true)
227+
logEvent(event)
228228
}
229229

230230
override def onNodeExcluded(event: SparkListenerNodeExcluded): Unit = {
231-
logEvent(event, flushLogger = true)
231+
logEvent(event)
232232
}
233233

234234
override def onNodeUnblacklisted(event: SparkListenerNodeUnblacklisted): Unit = {
235-
logEvent(event, flushLogger = true)
235+
logEvent(event)
236236
}
237237

238238
override def onNodeUnexcluded(event: SparkListenerNodeUnexcluded): Unit = {
239-
logEvent(event, flushLogger = true)
239+
logEvent(event)
240240
}
241241

242242
override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {
243243
if (shouldLogBlockUpdates) {
244-
logEvent(event, flushLogger = true)
244+
logEvent(event)
245245
}
246246
}
247247

@@ -266,12 +266,12 @@ private[spark] class EventLoggingListener(
266266
}
267267

268268
override def onResourceProfileAdded(event: SparkListenerResourceProfileAdded): Unit = {
269-
logEvent(event, flushLogger = true)
269+
logEvent(event)
270270
}
271271

272272
override def onOtherEvent(event: SparkListenerEvent): Unit = {
273273
if (event.logEvent) {
274-
logEvent(event, flushLogger = true)
274+
logEvent(event)
275275
}
276276
}
277277

0 commit comments

Comments
 (0)