Changes to allow re-use of test applications

Author: pwendell

dev/audit-release/audit_release.py

@@ -114,6 +114,7 @@ def ensure_path_not_present(x):
 os.environ["SPARK_VERSION"] = RELEASE_VERSION
 os.environ["SCALA_VERSION"] = SCALA_VERSION
 os.environ["SPARK_RELEASE_REPOSITORY"] = RELEASE_REPOSITORY
+os.environ["SPARK_AUDIT_MASTER"] = "local"
 for module in modules:
   os.environ["SPARK_MODULE"] = module
   ret = run_cmd("sbt clean update", exit_on_failure=False)

dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala

@@ -24,8 +24,13 @@ import org.apache.spark.SparkContext._
 
 object SimpleApp {
   def main(args: Array[String]) {
+    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
+      case Some(master) => new SparkConf().setAppName("Simple Spark App").setMaster(master)
+      case None => new SparkConf().setAppName("Simple Spark App")
+    }
     val logFile = "input.txt"
-    val sc = new SparkContext("local", "Simple App")
+    val sc = new SparkContext(conf)
+    SparkContext.jarOfClass(this.getClass).foreach(sc.addJar)
     val logData = sc.textFile(logFile, 2).cache()
     val numAs = logData.filter(line => line.contains("a")).count()
     val numBs = logData.filter(line => line.contains("b")).count()

dev/audit-release/sbt_app_graphx/src/main/scala/GraphxApp.scala

@@ -17,14 +17,20 @@
 
 package main.scala
 
-import org.apache.spark.SparkContext
+import org.apache.spark.{SparkContext, SparkConf}
 import org.apache.spark.SparkContext._
 import org.apache.spark.graphx._
 import org.apache.spark.rdd.RDD
 
 object GraphXApp {
   def main(args: Array[String]) {
-    val sc = new SparkContext("local", "Simple GraphX App")
+    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
+      case Some(master) => new SparkConf().setAppName("Simple GraphX App").setMaster(master)
+      case None => new SparkConf().setAppName("Simple Graphx App")
+    }
+    val sc = new SparkContext(conf)
+    SparkContext.jarOfClass(this.getClass).foreach(sc.addJar)
+
     val users: RDD[(VertexId, (String, String))] =
       sc.parallelize(Array((3L, ("rxin", "student")), (7L, ("jgonzal", "postdoc")),
                            (5L, ("franklin", "prof")), (2L, ("istoica", "prof")),

dev/audit-release/sbt_app_streaming/src/main/scala/StreamingApp.scala

@@ -27,10 +27,12 @@ import org.apache.spark.streaming._
 object SparkStreamingExample {
 
   def main(args: Array[String]) {
-    val conf = new SparkConf(true)
-      .setMaster("local[2]")
-      .setAppName("Streaming test")
+    val conf = sys.env.get("SPARK_AUDIT_MASTER") match {
+      case Some(master) => new SparkConf().setAppName("Simple Streaming App").setMaster(master)
+      case None => new SparkConf().setAppName("Simple Streaming App")
+    }
     val ssc = new StreamingContext(conf, Seconds(1))
+    SparkContext.jarOfClass(this.getClass).foreach(ssc.sparkContext.addJar)
     val seen = ListBuffer[RDD[Int]]()
 
     val rdd1 = ssc.sparkContext.makeRDD(1 to 100, 10)