updated per TD's feedback. simplified examples, updated docs

Author: cfregly

assembly/pom.xml

@@ -190,7 +190,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.spark</groupId>
-          <artifactId>kinesis-asl_${scala.binary.version}</artifactId>
+          <artifactId>spark-streaming-kinesis-asl_${scala.binary.version}</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>

bin/run-example

@@ -29,7 +29,9 @@ if [ -n "$1" ]; then
 else
   echo "Usage: ./bin/run-example <example-class> [example-args]" 1>&2
   echo "  - set MASTER=XX to use a specific master" 1>&2
-  echo "  - can use abbreviated example class name (e.g. SparkPi, mllib.LinearRegression)" 1>&2
+  echo "  - can use abbreviated example class name relative to com.apache.spark.examples" 1>&2
+  echo "     (e.g. SparkPi, mllib.LinearRegression, streaming.KinesisWordCountASL)" 1>&2
+  echo "  - to run the Kinesis Spark Streaming example, make sure you build with -Pkinesis-asl" 1>&2
   exit 1
 fi
 

dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala

@@ -47,14 +47,14 @@ object SimpleApp {
       System.exit(-1)
     }
     if (foundGanglia) {
-      println("Ganglia sink was loaded via spark-ganglia-lgpl")
+      println("Ganglia sink was loaded via spark-core")
       System.exit(-1)
     }
 
     // Remove kinesis from default build due to ASL license issue
     val foundKinesis = Try(Class.forName("org.apache.spark.streaming.kinesis.KinesisUtils")).isSuccess
     if (foundKinesis) {
-      println("Kinesis was loaded via kinesis-asl")
+      println("Kinesis was loaded via spark-core")
       System.exit(-1)
     }
   }

dev/audit-release/sbt_app_kinesis/build.sbt

@@ -23,7 +23,7 @@ scalaVersion := System.getenv.get("SCALA_VERSION")
 
 libraryDependencies += "org.apache.spark" %% "spark-core" % System.getenv.get("SPARK_VERSION")
 libraryDependencies += "org.apache.spark" %% "spark-streaming" % System.getenv.get("SPARK_VERSION")
-libraryDependencies += "org.apache.spark" %% "kinesis-asl" % System.getenv.get("SPARK_VERSION")
+libraryDependencies += "org.apache.spark" %% "spark-streaming-kinesis-asl" % System.getenv.get("SPARK_VERSION")
 
 resolvers ++= Seq(
   "Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),

docs/streaming-kinesis.md

@@ -16,6 +16,7 @@ Deployment and runtime notes:
 <li>Each shard of a stream is processed by one or more KinesisReceiver's managed by the Kinesis Client Library (KCL) Worker.</li>
 <li>Said differently, a single KinesisReceiver can process many shards of a stream.</li>
 <li>You never need more KinesisReceivers than the number of shards in your stream.</li>
+<li>You can horizontally scale the receiving by creating more KinesisReceiver/DStreams (up to the number of shards for a given stream)</li>
 <li>The Kinesis assembly jar must also be present on all worker nodes, as they will need access to the Kinesis Client Library.</li>
 <li>/tmp/checkpoint is a valid and accessible directory on all workers (or locally if running in local mode)</li>
 <li>This code uses the DefaultAWSCredentialsProviderChain and searches for credentials in the following order of precedence:<br/>

examples/pom.xml

@@ -98,7 +98,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>kinesis-asl_${scala.binary.version}</artifactId>
+      <artifactId>spark-streaming-kinesis-asl_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>