Set a configurable limit on maximum call stack depth. It can be useful in memory-constrained situations with large numbers of stages.

darabos · darabos · commit 6182da6d995d · 2014-06-06T15:51:18.000+02:00
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -840,9 +840,10 @@ private[spark] object Utils extends Logging {
         userCallStack += el.toString
       }
     }
+    val userCallStackDepth = System.getProperty("spark.callstack.depth", "20").toInt
     CallSite(
       short = "%s at %s:%s".format(lastSparkMethod, firstUserFile, firstUserLine),
-      long = userCallStack.mkString("\n"))
+      long = userCallStack.take(userCallStackDepth).mkString("\n"))
   }
 
   /** Return a string containing part of a file from byte 'start' to 'end'. */

Original file line number	Diff line number	Diff line change
`@@ -840,9 +840,10 @@ private[spark] object Utils extends Logging {`
`840`	`840`	`userCallStack += el.toString`
`841`	`841`	`}`
`842`	`842`	`}`
	`843`	`+ val userCallStackDepth = System.getProperty("spark.callstack.depth", "20").toInt`
`843`	`844`	`CallSite(`
`844`	`845`	`short = "%s at %s:%s".format(lastSparkMethod, firstUserFile, firstUserLine),`
`845`		`- long = userCallStack.mkString("\n"))`
	`846`	`+ long = userCallStack.take(userCallStackDepth).mkString("\n"))`
`846`	`847`	`}`
`847`	`848`
`848`	`849`	`/** Return a string containing part of a file from byte 'start' to 'end'. */`