Skip to content

Commit 0476e5e

Browse files
ScrapCodesScrapCodes
committed
Scala 2.11 support with repl and all build changes.
1 parent f73b56f commit 0476e5e

40 files changed

+4005
-104
lines changed

‎.rat-excludes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ SparkImports.scala
4444
SparkJLineCompletion.scala
4545
SparkJLineReader.scala
4646
SparkMemberHandlers.scala
47+
SparkReplReporter.scala
4748
sbt
4849
sbt-launch-lib.bash
4950
plugins.sbt

‎assembly/pom.xml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,22 @@
6666
</dependency>
6767
<dependency>
6868
<groupId>org.apache.spark</groupId>
69-
<artifactId>spark-repl_${scala.binary.version}</artifactId>
69+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
7070
<version>${project.version}</version>
7171
</dependency>
7272
<dependency>
7373
<groupId>org.apache.spark</groupId>
74-
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
74+
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
7575
<version>${project.version}</version>
7676
</dependency>
7777
<dependency>
7878
<groupId>org.apache.spark</groupId>
79-
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
79+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
8080
<version>${project.version}</version>
8181
</dependency>
8282
<dependency>
8383
<groupId>org.apache.spark</groupId>
84-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
84+
<artifactId>spark-repl_${scala.binary.version}</artifactId>
8585
<version>${project.version}</version>
8686
</dependency>
8787
</dependencies>

‎bin/compute-classpath.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
# This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
2121
# script and the ExecutorRunner in standalone cluster mode.
2222

23-
SCALA_VERSION=2.10
23+
SCALA_VERSION=${SCALA_VERSION:-"2.10"}
2424

2525
# Figure out where Spark is installed
2626
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
@@ -121,6 +121,9 @@ if [ -n "$datanucleus_jars" ]; then
121121
fi
122122
fi
123123

124+
test_jars=$(find "$FWDIR"/lib_managed/test \( -name '*jar' -a -type f \) 2>/dev/null | \
125+
tr "\n" : | sed s/:$//g)
126+
124127
# Add test classes if we're running from SBT or Maven with SPARK_TESTING set to 1
125128
if [[ $SPARK_TESTING == 1 ]]; then
126129
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/test-classes"
@@ -132,6 +135,9 @@ if [[ $SPARK_TESTING == 1 ]]; then
132135
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/test-classes"
133136
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/test-classes"
134137
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/test-classes"
138+
if [[ $SCALA_VERSION == "2.11" ]]; then
139+
CLASSPATH="$CLASSPATH:$test_jars"
140+
fi
135141
fi
136142

137143
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail !

‎conf/spark-env.sh.template‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@
33
# This file is sourced when running various Spark programs.
44
# Copy it as spark-env.sh and edit that to configure Spark for your site.
55

6+
# Uncomment this if you plan to use scala 2.11
7+
# SCALA_VERSION=2.11
8+
69
# Options read when launching programs locally with
710
# ./bin/run-example or ./bin/spark-submit
811
# - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files

‎core/pom.xml‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,34 @@
3434
<name>Spark Project Core</name>
3535
<url>http://spark.apache.org/</url>
3636
<dependencies>
37+
<dependency>
38+
<groupId>com.twitter</groupId>
39+
<artifactId>chill_${scala.binary.version}</artifactId>
40+
<exclusions>
41+
<exclusion>
42+
<groupId>org.ow2.asm</groupId>
43+
<artifactId>asm</artifactId>
44+
</exclusion>
45+
<exclusion>
46+
<groupId>org.ow2.asm</groupId>
47+
<artifactId>asm-commons</artifactId>
48+
</exclusion>
49+
</exclusions>
50+
</dependency>
51+
<dependency>
52+
<groupId>com.twitter</groupId>
53+
<artifactId>chill-java</artifactId>
54+
<exclusions>
55+
<exclusion>
56+
<groupId>org.ow2.asm</groupId>
57+
<artifactId>asm</artifactId>
58+
</exclusion>
59+
<exclusion>
60+
<groupId>org.ow2.asm</groupId>
61+
<artifactId>asm-commons</artifactId>
62+
</exclusion>
63+
</exclusions>
64+
</dependency>
3765
<dependency>
3866
<groupId>org.apache.hadoop</groupId>
3967
<artifactId>hadoop-client</artifactId>
@@ -132,14 +160,6 @@
132160
<groupId>net.jpountz.lz4</groupId>
133161
<artifactId>lz4</artifactId>
134162
</dependency>
135-
<dependency>
136-
<groupId>com.twitter</groupId>
137-
<artifactId>chill_${scala.binary.version}</artifactId>
138-
</dependency>
139-
<dependency>
140-
<groupId>com.twitter</groupId>
141-
<artifactId>chill-java</artifactId>
142-
</dependency>
143163
<dependency>
144164
<groupId>org.roaringbitmap</groupId>
145165
<artifactId>RoaringBitmap</artifactId>
@@ -276,6 +296,10 @@
276296
<artifactId>easymockclassextension</artifactId>
277297
<scope>test</scope>
278298
</dependency>
299+
<dependency>
300+
<groupId>com.twitter</groupId>
301+
<artifactId>chill-java</artifactId>
302+
</dependency>
279303
<dependency>
280304
<groupId>asm</groupId>
281305
<artifactId>asm</artifactId>
@@ -424,4 +448,5 @@
424448
</resource>
425449
</resources>
426450
</build>
451+
427452
</project>

‎dev/change-version-to-2.10.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
find -name 'pom.xml' -exec sed -i 's|\(artifactId.*\)_2.11|\1_2.10|g' {} \;

‎dev/change-version-to-2.11.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
find -name 'pom.xml' -exec sed -i 's|\(artifactId.*\)_2.10|\1_2.11|g' {} \;

‎examples/pom.xml‎

Lines changed: 120 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -34,24 +34,6 @@
3434
<name>Spark Project Examples</name>
3535
<url>http://spark.apache.org/</url>
3636

37-
<profiles>
38-
<profile>
39-
<id>kinesis-asl</id>
40-
<dependencies>
41-
<dependency>
42-
<groupId>org.apache.spark</groupId>
43-
<artifactId>spark-streaming-kinesis-asl_${scala.binary.version}</artifactId>
44-
<version>${project.version}</version>
45-
</dependency>
46-
<dependency>
47-
<groupId>org.apache.httpcomponents</groupId>
48-
<artifactId>httpclient</artifactId>
49-
<version>${commons.httpclient.version}</version>
50-
</dependency>
51-
</dependencies>
52-
</profile>
53-
</profiles>
54-
5537
<dependencies>
5638
<!-- Promote Guava to compile scope in this module so it's included while shading. -->
5739
<dependency>
@@ -100,24 +82,19 @@
10082
<artifactId>spark-streaming-twitter_${scala.binary.version}</artifactId>
10183
<version>${project.version}</version>
10284
</dependency>
103-
<dependency>
104-
<groupId>org.apache.spark</groupId>
105-
<artifactId>spark-streaming-kafka_${scala.binary.version}</artifactId>
106-
<version>${project.version}</version>
107-
</dependency>
10885
<dependency>
10986
<groupId>org.apache.spark</groupId>
11087
<artifactId>spark-streaming-flume_${scala.binary.version}</artifactId>
11188
<version>${project.version}</version>
11289
</dependency>
11390
<dependency>
11491
<groupId>org.apache.spark</groupId>
115-
<artifactId>spark-streaming-zeromq_${scala.binary.version}</artifactId>
92+
<artifactId>spark-streaming-mqtt_${scala.binary.version}</artifactId>
11693
<version>${project.version}</version>
11794
</dependency>
11895
<dependency>
11996
<groupId>org.apache.spark</groupId>
120-
<artifactId>spark-streaming-mqtt_${scala.binary.version}</artifactId>
97+
<artifactId>spark-streaming-zeromq_${scala.binary.version}</artifactId>
12198
<version>${project.version}</version>
12299
</dependency>
123100
<dependency>
@@ -152,10 +129,6 @@
152129
<artifactId>jetty-server</artifactId>
153130
</dependency>
154131
<dependency>
155-
<groupId>com.twitter</groupId>
156-
<artifactId>algebird-core_${scala.binary.version}</artifactId>
157-
<version>0.1.11</version>
158-
</dependency>
159132
<dependency>
160133
<groupId>org.apache.commons</groupId>
161134
<artifactId>commons-math3</artifactId>
@@ -292,4 +265,122 @@
292265
</plugin>
293266
</plugins>
294267
</build>
268+
<profiles>
269+
<profile>
270+
<id>kinesis-asl</id>
271+
<dependencies>
272+
<dependency>
273+
<groupId>org.apache.spark</groupId>
274+
<artifactId>spark-streaming-kinesis-asl_${scala.binary.version}</artifactId>
275+
<version>${project.version}</version>
276+
</dependency>
277+
<dependency>
278+
<groupId>org.apache.httpcomponents</groupId>
279+
<artifactId>httpclient</artifactId>
280+
<version>${commons.httpclient.version}</version>
281+
</dependency>
282+
</dependencies>
283+
</profile>
284+
<profile>
285+
<id>scala-2.10</id>
286+
<activation>
287+
<activeByDefault>true</activeByDefault>
288+
</activation>
289+
<dependencies>
290+
<dependency>
291+
<groupId>org.apache.spark</groupId>
292+
<artifactId>spark-streaming-kafka_${scala.binary.version}</artifactId>
293+
<version>${project.version}</version>
294+
</dependency>
295+
<dependency>
296+
<groupId>com.twitter</groupId>
297+
<artifactId>algebird-core_${scala.binary.version}</artifactId>
298+
<version>0.1.11</version>
299+
</dependency>
300+
</dependencies>
301+
<build>
302+
<plugins>
303+
<plugin>
304+
<groupId>org.codehaus.mojo</groupId>
305+
<artifactId>build-helper-maven-plugin</artifactId>
306+
<executions>
307+
<execution>
308+
<id>add-scala-sources</id>
309+
<phase>generate-sources</phase>
310+
<goals>
311+
<goal>add-source</goal>
312+
</goals>
313+
<configuration>
314+
<sources>
315+
<source>src/main/scala</source>
316+
<source>scala-2.10/src/main/scala</source>
317+
<source>scala-2.10/src/main/java</source>
318+
</sources>
319+
</configuration>
320+
</execution>
321+
<execution>
322+
<id>add-scala-test-sources</id>
323+
<phase>generate-test-sources</phase>
324+
<goals>
325+
<goal>add-test-source</goal>
326+
</goals>
327+
<configuration>
328+
<sources>
329+
<source>src/test/scala</source>
330+
<source>scala-2.10/src/test/scala</source>
331+
<source>scala-2.10/src/test/java</source>
332+
</sources>
333+
</configuration>
334+
</execution>
335+
</executions>
336+
</plugin>
337+
</plugins>
338+
</build>
339+
</profile>
340+
<profile>
341+
<id>scala-2.11</id>
342+
<activation>
343+
<activeByDefault>false</activeByDefault>
344+
</activation>
345+
<dependencies>
346+
<!-- Streaming Kafka and zeromq modules are disabled for now. -->
347+
</dependencies>
348+
<build>
349+
<plugins>
350+
<plugin>
351+
<groupId>org.codehaus.mojo</groupId>
352+
<artifactId>build-helper-maven-plugin</artifactId>
353+
<executions>
354+
<execution>
355+
<id>add-scala-sources</id>
356+
<phase>generate-sources</phase>
357+
<goals>
358+
<goal>add-source</goal>
359+
</goals>
360+
<configuration>
361+
<sources>
362+
<source>src/main/scala</source>
363+
<source>scala-2.11/src/main/scala</source>
364+
</sources>
365+
</configuration>
366+
</execution>
367+
<execution>
368+
<id>add-scala-test-sources</id>
369+
<phase>generate-test-sources</phase>
370+
<goals>
371+
<goal>add-test-source</goal>
372+
</goals>
373+
<configuration>
374+
<sources>
375+
<source>src/test/scala</source>
376+
<source>scala-2.11/src/test/scala</source>
377+
</sources>
378+
</configuration>
379+
</execution>
380+
</executions>
381+
</plugin>
382+
</plugins>
383+
</build>
384+
</profile>
385+
</profiles>
295386
</project>

‎examples/src/main/java/org/apache/spark/examples/streaming/JavaKafkaWordCount.java‎ renamed to ‎examples/scala-2.10/src/main/java/org/apache/spark/examples/streaming/JavaKafkaWordCount.java‎

File renamed without changes.

‎examples/src/main/scala/org/apache/spark/examples/streaming/KafkaWordCount.scala‎ renamed to ‎examples/scala-2.10/src/main/scala/org/apache/spark/examples/streaming/KafkaWordCount.scala‎

File renamed without changes.

0 commit comments

Comments
 (0)