[SPARK-1853] Show Streaming application code context (file, line number) in Spark Stages UI

Author: mubarak

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -1296,9 +1296,6 @@ object SparkContext extends Logging {
 
   private[spark] val SPARK_UNKNOWN_USER = "<unknown>"
 
-  private[spark] val SPARK_JOB_CALL_SITE_SHORT = "spark.job.callSiteShort"
-  private[spark] val SPARK_JOB_CALL_SITE_LONG = "spark.job.callSiteLong"
-
   implicit object DoubleAccumulatorParam extends AccumulatorParam[Double] {
     def addInPlace(t1: Double, t2: Double): Double = t1 + t2
     def zero(initialValue: Double) = 0.0

core/src/main/scala/org/apache/spark/util/Utils.scala

@@ -852,7 +852,7 @@ private[spark] object Utils extends Logging {
         callStack += el.toString
       }
     }
-    val callStackDepth = System.getProperty("spark.callstack.depth", "10").toInt
+    val callStackDepth = System.getProperty("spark.callstack.depth", "20").toInt
     CallSite(
       short = "%s at %s:%s".format(lastSparkMethod, firstUserFile, firstUserLine),
       long = callStack.take(callStackDepth).mkString("\n"))

streaming/src/main/scala/org/apache/spark/streaming/dstream/DStream.scala

@@ -322,7 +322,6 @@ abstract class DStream[T: ClassTag] (
   private[streaming] def generateJob(time: Time): Option[Job] = {
     getOrCompute(time) match {
       case Some(rdd) => {
-        //ssc.sc.setJobGroup("g","d")
         val jobFunc = () => {
           val emptyFunc = { (iterator: Iterator[T]) => {} }
           context.sparkContext.runJob(rdd, emptyFunc)