From 292746d03cc132a64ffe66e4f4276ff54f3cfa56 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Wed, 18 Jun 2014 19:27:53 -0700 Subject: [PATCH] SPARK-2156: Provide more overhead for Akka content in result message. Previously we only allowed 1 KB of overhead, but in practice it seems like Akka can insert dozens of KB of overhead. This patch gives more slack, avoiding the case where we exceed the max frame size and messages are dropped silently. --- core/src/main/scala/org/apache/spark/executor/Executor.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala index baee7a216a7c..07fe3ee07dc0 100644 --- a/core/src/main/scala/org/apache/spark/executor/Executor.scala +++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala @@ -212,7 +212,9 @@ private[spark] class Executor( val serializedDirectResult = ser.serialize(directResult) logInfo("Serialized size of result for " + taskId + " is " + serializedDirectResult.limit) val serializedResult = { - if (serializedDirectResult.limit >= akkaFrameSize - 1024) { + // space for akka headers and other arguments + val akkaOverhead = 100 * 1024 + if (serializedDirectResult.limit >= akkaFrameSize - akkaOverhead) { logInfo("Storing result for " + taskId + " in local BlockManager") val blockId = TaskResultBlockId(taskId) env.blockManager.putBytes(