From 946351ce11f9178e952c72f23dd4f93891a15c15 Mon Sep 17 00:00:00 2001 From: Hong Shen Date: Thu, 13 Nov 2014 20:38:09 +0800 Subject: [PATCH 1/5] Edit spilling log from MB to B --- .../scala/org/apache/spark/util/collection/Spillable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index d7dccd4af8c6e..5e29a81e48197 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -105,7 +105,7 @@ private[spark] trait Spillable[C] { */ @inline private def logSpillage(size: Long) { val threadId = Thread.currentThread().getId - logInfo("Thread %d spilling in-memory map of %d MB to disk (%d time%s so far)" - .format(threadId, size / (1024 * 1024), _spillCount, if (_spillCount > 1) "s" else "")) + logInfo("Thread %d spilling in-memory map of %d B to disk (%d time%s so far)" + .format(threadId, size, _spillCount, if (_spillCount > 1) "s" else "")) } } From da761c27c75b9644ecc8240e72039a2fca02e2f2 Mon Sep 17 00:00:00 2001 From: Hong Shen Date: Fri, 14 Nov 2014 09:11:53 +0800 Subject: [PATCH 2/5] Edit spilling log from MB to B --- .../scala/org/apache/spark/util/collection/Spillable.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index 5e29a81e48197..642feaeb925b7 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -19,6 +19,7 @@ package org.apache.spark.util.collection import org.apache.spark.Logging import org.apache.spark.SparkEnv +import org.apache.spark.util.Utils /** * Spills contents of an in-memory collection to disk when the memory threshold @@ -106,6 +107,6 @@ private[spark] trait Spillable[C] { @inline private def logSpillage(size: Long) { val threadId = Thread.currentThread().getId logInfo("Thread %d spilling in-memory map of %d B to disk (%d time%s so far)" - .format(threadId, size, _spillCount, if (_spillCount > 1) "s" else "")) + .format(threadId, Utils.bytesToString(size), _spillCount, if (_spillCount > 1) "s" else "")) } } From e9145e8ac6798bb9e2587e2eb67da6209456840f Mon Sep 17 00:00:00 2001 From: Hong Shen Date: Fri, 14 Nov 2014 09:14:45 +0800 Subject: [PATCH 3/5] Edit spilling log from MB to B --- .../main/scala/org/apache/spark/util/collection/Spillable.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index 642feaeb925b7..c8be43b2a314f 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -106,7 +106,7 @@ private[spark] trait Spillable[C] { */ @inline private def logSpillage(size: Long) { val threadId = Thread.currentThread().getId - logInfo("Thread %d spilling in-memory map of %d B to disk (%d time%s so far)" + logInfo("Thread %d spilling in-memory map of %s to disk (%d time%s so far)" .format(threadId, Utils.bytesToString(size), _spillCount, if (_spillCount > 1) "s" else "")) } } From 21ee960d5aaac68f14ebac49c4f0d947f9209f49 Mon Sep 17 00:00:00 2001 From: Hong Shen Date: Fri, 14 Nov 2014 09:24:19 +0800 Subject: [PATCH 4/5] Edit spilling log from MB to B --- .../scala/org/apache/spark/util/collection/Spillable.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index c8be43b2a314f..1c3eb7e5b7bb1 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -19,7 +19,6 @@ package org.apache.spark.util.collection import org.apache.spark.Logging import org.apache.spark.SparkEnv -import org.apache.spark.util.Utils /** * Spills contents of an in-memory collection to disk when the memory threshold @@ -107,6 +106,6 @@ private[spark] trait Spillable[C] { @inline private def logSpillage(size: Long) { val threadId = Thread.currentThread().getId logInfo("Thread %d spilling in-memory map of %s to disk (%d time%s so far)" - .format(threadId, Utils.bytesToString(size), _spillCount, if (_spillCount > 1) "s" else "")) + .format(threadId, org.apache.spark.util.Utils.bytesToString(size), _spillCount, if (_spillCount > 1) "s" else "")) } } From 4653378fb6addfdf4fb21e4e75570c163d601bfb Mon Sep 17 00:00:00 2001 From: Hong Shen Date: Fri, 14 Nov 2014 09:26:33 +0800 Subject: [PATCH 5/5] Edit spilling log from MB to B --- .../scala/org/apache/spark/util/collection/Spillable.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala index 1c3eb7e5b7bb1..0e4c6d633a4a9 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala @@ -106,6 +106,7 @@ private[spark] trait Spillable[C] { @inline private def logSpillage(size: Long) { val threadId = Thread.currentThread().getId logInfo("Thread %d spilling in-memory map of %s to disk (%d time%s so far)" - .format(threadId, org.apache.spark.util.Utils.bytesToString(size), _spillCount, if (_spillCount > 1) "s" else "")) + .format(threadId, org.apache.spark.util.Utils.bytesToString(size), + _spillCount, if (_spillCount > 1) "s" else "")) } }