diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala index 963ed121547c..f3bb6d5af335 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala @@ -21,7 +21,7 @@ import java.io._ import java.net.URI import java.nio.charset.StandardCharsets -import org.apache.commons.compress.utils.CountingOutputStream +import org.apache.commons.io.output.CountingOutputStream import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataOutputStream, Path} import org.apache.hadoop.fs.permission.FsPermission @@ -330,7 +330,7 @@ class RollingEventLogFilesWriter( override def writeEvent(eventJson: String, flushLogger: Boolean = false): Unit = { writer.foreach { w => - val currentLen = countingOutputStream.get.getBytesWritten + val currentLen = countingOutputStream.get.getByteCount if (currentLen + eventJson.length > eventFileMaxLength) { rollEventLogFile() }