Skip to content

Commit 3689214

Browse files
JoshRosenJoshRosen
committed
Merge remote-tracking branch 'origin/master' into SPARK-1600
2 parents c8f06b1 + 5c506ce commit 3689214

File tree

178 files changed

+3427
-866
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

178 files changed

+3427
-866
lines changed

‎.gitignore‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,19 @@
88
*.pyc
99
.idea/
1010
.idea_modules/
11-
sbt/*.jar
11+
build/*.jar
1212
.settings
1313
.cache
14+
cache
1415
.generated-mima*
15-
/build/
1616
work/
1717
out/
1818
.DS_Store
1919
third_party/libmesos.so
2020
third_party/libmesos.dylib
21+
build/apache-maven*
22+
build/zinc*
23+
build/scala*
2124
conf/java-opts
2225
conf/*.sh
2326
conf/*.cmd

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ To build Spark and its example programs, run:
2626

2727
(You do not need to do this if you downloaded a pre-built package.)
2828
More detailed documentation is available from the project site, at
29-
["Building Spark with Maven"](http://spark.apache.org/docs/latest/building-with-maven.html).
29+
["Building Spark with Maven"](http://spark.apache.org/docs/latest/building-spark.html).
3030

3131
## Interactive Scala Shell
3232

‎build/mvn‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# Determine the current working directory
21+
_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
22+
# Preserve the calling directory
23+
_CALLING_DIR="$(pwd)"
24+
25+
# Installs any application tarball given a URL, the expected tarball name,
26+
# and, optionally, a checkable binary path to determine if the binary has
27+
# already been installed
28+
## Arg1 - URL
29+
## Arg2 - Tarball Name
30+
## Arg3 - Checkable Binary
31+
install_app() {
32+
local remote_tarball="$1/$2"
33+
local local_tarball="${_DIR}/$2"
34+
local binary="${_DIR}/$3"
35+
36+
# setup `curl` and `wget` silent options if we're running on Jenkins
37+
local curl_opts=""
38+
local wget_opts=""
39+
if [ -n "$AMPLAB_JENKINS" ]; then
40+
curl_opts="-s"
41+
wget_opts="--quiet"
42+
else
43+
curl_opts="--progress-bar"
44+
wget_opts="--progress=bar:force"
45+
fi
46+
47+
if [ -z "$3" -o ! -f "$binary" ]; then
48+
# check if we already have the tarball
49+
# check if we have curl installed
50+
# download application
51+
[ ! -f "${local_tarball}" ] && [ -n "`which curl 2>/dev/null`" ] && \
52+
echo "exec: curl ${curl_opts} ${remote_tarball}" && \
53+
curl ${curl_opts} "${remote_tarball}" > "${local_tarball}"
54+
# if the file still doesn't exist, lets try `wget` and cross our fingers
55+
[ ! -f "${local_tarball}" ] && [ -n "`which wget 2>/dev/null`" ] && \
56+
echo "exec: wget ${wget_opts} ${remote_tarball}" && \
57+
wget ${wget_opts} -O "${local_tarball}" "${remote_tarball}"
58+
# if both were unsuccessful, exit
59+
[ ! -f "${local_tarball}" ] && \
60+
echo -n "ERROR: Cannot download $2 with cURL or wget; " && \
61+
echo "please install manually and try again." && \
62+
exit 2
63+
cd "${_DIR}" && tar -xzf "$2"
64+
rm -rf "$local_tarball"
65+
fi
66+
}
67+
68+
# Install maven under the build/ folder
69+
install_mvn() {
70+
install_app \
71+
"http://apache.claz.org/maven/maven-3/3.2.3/binaries" \
72+
"apache-maven-3.2.3-bin.tar.gz" \
73+
"apache-maven-3.2.3/bin/mvn"
74+
MVN_BIN="${_DIR}/apache-maven-3.2.3/bin/mvn"
75+
}
76+
77+
# Install zinc under the build/ folder
78+
install_zinc() {
79+
local zinc_path="zinc-0.3.5.3/bin/zinc"
80+
[ ! -f "${zinc_path}" ] && ZINC_INSTALL_FLAG=1
81+
install_app \
82+
"http://downloads.typesafe.com/zinc/0.3.5.3" \
83+
"zinc-0.3.5.3.tgz" \
84+
"${zinc_path}"
85+
ZINC_BIN="${_DIR}/${zinc_path}"
86+
}
87+
88+
# Determine the Scala version from the root pom.xml file, set the Scala URL,
89+
# and, with that, download the specific version of Scala necessary under
90+
# the build/ folder
91+
install_scala() {
92+
# determine the Scala version used in Spark
93+
local scala_version=`grep "scala.version" "${_DIR}/../pom.xml" | \
94+
head -1 | cut -f2 -d'>' | cut -f1 -d'<'`
95+
local scala_bin="${_DIR}/scala-${scala_version}/bin/scala"
96+
97+
install_app \
98+
"http://downloads.typesafe.com/scala/${scala_version}" \
99+
"scala-${scala_version}.tgz" \
100+
"scala-${scala_version}/bin/scala"
101+
102+
SCALA_COMPILER="$(cd "$(dirname ${scala_bin})/../lib" && pwd)/scala-compiler.jar"
103+
SCALA_LIBRARY="$(cd "$(dirname ${scala_bin})/../lib" && pwd)/scala-library.jar"
104+
}
105+
106+
# Determines if a given application is already installed. If not, will attempt
107+
# to install
108+
## Arg1 - application name
109+
## Arg2 - Alternate path to local install under build/ dir
110+
check_and_install_app() {
111+
# create the local environment variable in uppercase
112+
local app_bin="`echo $1 | awk '{print toupper(\$0)}'`_BIN"
113+
# some black magic to set the generated app variable (i.e. MVN_BIN) into the
114+
# environment
115+
eval "${app_bin}=`which $1 2>/dev/null`"
116+
117+
if [ -z "`which $1 2>/dev/null`" ]; then
118+
install_$1
119+
fi
120+
}
121+
122+
# Setup healthy defaults for the Zinc port if none were provided from
123+
# the environment
124+
ZINC_PORT=${ZINC_PORT:-"3030"}
125+
126+
# Check and install all applications necessary to build Spark
127+
check_and_install_app "mvn"
128+
129+
# Install the proper version of Scala and Zinc for the build
130+
install_zinc
131+
install_scala
132+
133+
# Reset the current working directory
134+
cd "${_CALLING_DIR}"
135+
136+
# Now that zinc is ensured to be installed, check its status and, if its
137+
# not running or just installed, start it
138+
if [ -n "${ZINC_INSTALL_FLAG}" -o -z "`${ZINC_BIN} -status`" ]; then
139+
${ZINC_BIN} -shutdown
140+
${ZINC_BIN} -start -port ${ZINC_PORT} \
141+
-scala-compiler "${SCALA_COMPILER}" \
142+
-scala-library "${SCALA_LIBRARY}" &>/dev/null
143+
fi
144+
145+
# Set any `mvn` options if not already present
146+
export MAVEN_OPTS=${MAVEN_OPTS:-"-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"}
147+
148+
# Last, call the `mvn` command as usual
149+
${MVN_BIN} "$@"

‎build/sbt‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
# When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so
21+
# that we can run Hive to generate the golden answer. This is not required for normal development
22+
# or testing.
23+
for i in "$HIVE_HOME"/lib/*
24+
do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i"
25+
done
26+
export HADOOP_CLASSPATH
27+
28+
realpath () {
29+
(
30+
TARGET_FILE="$1"
31+
32+
cd "$(dirname "$TARGET_FILE")"
33+
TARGET_FILE="$(basename "$TARGET_FILE")"
34+
35+
COUNT=0
36+
while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
37+
do
38+
TARGET_FILE="$(readlink "$TARGET_FILE")"
39+
cd $(dirname "$TARGET_FILE")
40+
TARGET_FILE="$(basename $TARGET_FILE)"
41+
COUNT=$(($COUNT + 1))
42+
done
43+
44+
echo "$(pwd -P)/"$TARGET_FILE""
45+
)
46+
}
47+
48+
. "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash
49+
50+
51+
declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
52+
declare -r sbt_opts_file=".sbtopts"
53+
declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
54+
55+
usage() {
56+
cat <<EOM
57+
Usage: $script_name [options]
58+
59+
-h | -help print this message
60+
-v | -verbose this runner is chattier
61+
-d | -debug set sbt log level to debug
62+
-no-colors disable ANSI color codes
63+
-sbt-create start sbt even if current directory contains no sbt project
64+
-sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt)
65+
-sbt-boot <path> path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
66+
-ivy <path> path to local Ivy repository (default: ~/.ivy2)
67+
-mem <integer> set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
68+
-no-share use all local caches; no sharing
69+
-no-global uses global caches, but does not use global ~/.sbt directory.
70+
-jvm-debug <port> Turn on JVM debugging, open at the given port.
71+
-batch Disable interactive mode
72+
73+
# sbt version (default: from project/build.properties if present, else latest release)
74+
-sbt-version <version> use the specified version of sbt
75+
-sbt-jar <path> use the specified jar as the sbt launcher
76+
-sbt-rc use an RC version of sbt
77+
-sbt-snapshot use a snapshot version of sbt
78+
79+
# java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
80+
-java-home <path> alternate JAVA_HOME
81+
82+
# jvm options and output control
83+
JAVA_OPTS environment variable, if unset uses "$java_opts"
84+
SBT_OPTS environment variable, if unset uses "$default_sbt_opts"
85+
.sbtopts if this file exists in the current directory, it is
86+
prepended to the runner args
87+
/etc/sbt/sbtopts if this file exists, it is prepended to the runner args
88+
-Dkey=val pass -Dkey=val directly to the java runtime
89+
-J-X pass option -X directly to the java runtime
90+
(-J is stripped)
91+
-S-X add -X to sbt's scalacOptions (-S is stripped)
92+
-PmavenProfiles Enable a maven profile for the build.
93+
94+
In the case of duplicated or conflicting options, the order above
95+
shows precedence: JAVA_OPTS lowest, command line options highest.
96+
EOM
97+
}
98+
99+
process_my_args () {
100+
while [[ $# -gt 0 ]]; do
101+
case "$1" in
102+
-no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
103+
-no-share) addJava "$noshare_opts" && shift ;;
104+
-no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;;
105+
-sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
106+
-sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;;
107+
-debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
108+
-batch) exec </dev/null && shift ;;
109+
110+
-sbt-create) sbt_create=true && shift ;;
111+
112+
*) addResidual "$1" && shift ;;
113+
esac
114+
done
115+
116+
# Now, ensure sbt version is used.
117+
[[ "${sbt_version}XXX" != "XXX" ]] && addJava "-Dsbt.version=$sbt_version"
118+
}
119+
120+
loadConfigFile() {
121+
cat "$1" | sed '/^\#/d'
122+
}
123+
124+
# if sbtopts files exist, prepend their contents to $@ so it can be processed by this runner
125+
[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
126+
[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
127+
128+
run "$@"

‎sbt/sbt-launch-lib.bash‎ renamed to ‎build/sbt-launch-lib.bash‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ dlog () {
3737
}
3838

3939
acquire_sbt_jar () {
40-
SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
40+
SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
4141
URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
4242
URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
43-
JAR=sbt/sbt-launch-${SBT_VERSION}.jar
43+
JAR=build/sbt-launch-${SBT_VERSION}.jar
4444

4545
sbt_jar=$JAR
4646

@@ -150,7 +150,7 @@ process_args () {
150150
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && export JAVA_HOME=$2 && shift 2 ;;
151151

152152
-D*) addJava "$1" && shift ;;
153-
-J*) addJava "${1:2}" && shift ;;
153+
-J*) addJava "${1:2}" && shift ;;
154154
-P*) enableProfile "$1" && shift ;;
155155
*) addResidual "$1" && shift ;;
156156
esac

‎core/src/main/scala/org/apache/spark/MapOutputTracker.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ private[spark] class MapOutputTrackerMasterActor(tracker: MapOutputTrackerMaster
7676
*/
7777
private[spark] abstract class MapOutputTracker(conf: SparkConf) extends Logging {
7878
private val timeout = AkkaUtils.askTimeout(conf)
79+
private val retryAttempts = AkkaUtils.numRetries(conf)
80+
private val retryIntervalMs = AkkaUtils.retryWaitMs(conf)
7981

8082
/** Set to the MapOutputTrackerActor living on the driver. */
8183
var trackerActor: ActorRef = _
@@ -108,8 +110,7 @@ private[spark] abstract class MapOutputTracker(conf: SparkConf) extends Logging
108110
*/
109111
protected def askTracker(message: Any): Any = {
110112
try {
111-
val future = trackerActor.ask(message)(timeout)
112-
Await.result(future, timeout)
113+
AkkaUtils.askWithReply(message, trackerActor, retryAttempts, retryIntervalMs, timeout)
113114
} catch {
114115
case e: Exception =>
115116
logError("Error communicating with MapOutputTracker", e)

‎core/src/main/scala/org/apache/spark/SparkContext.scala‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -329,8 +329,13 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
329329
try {
330330
dagScheduler = new DAGScheduler(this)
331331
} catch {
332-
case e: Exception => throw
333-
new SparkException("DAGScheduler cannot be initialized due to %s".format(e.getMessage))
332+
case e: Exception => {
333+
try {
334+
stop()
335+
} finally {
336+
throw new SparkException("Error while constructing DAGScheduler", e)
337+
}
338+
}
334339
}
335340

336341
// start TaskScheduler after taskScheduler sets DAGScheduler reference in DAGScheduler's
@@ -1708,19 +1713,19 @@ object SparkContext extends Logging {
17081713

17091714
// Implicit conversions to common Writable types, for saveAsSequenceFile
17101715

1711-
implicit def intToIntWritable(i: Int) = new IntWritable(i)
1716+
implicit def intToIntWritable(i: Int): IntWritable = new IntWritable(i)
17121717

1713-
implicit def longToLongWritable(l: Long) = new LongWritable(l)
1718+
implicit def longToLongWritable(l: Long): LongWritable = new LongWritable(l)
17141719

1715-
implicit def floatToFloatWritable(f: Float) = new FloatWritable(f)
1720+
implicit def floatToFloatWritable(f: Float): FloatWritable = new FloatWritable(f)
17161721

1717-
implicit def doubleToDoubleWritable(d: Double) = new DoubleWritable(d)
1722+
implicit def doubleToDoubleWritable(d: Double): DoubleWritable = new DoubleWritable(d)
17181723

1719-
implicit def boolToBoolWritable (b: Boolean) = new BooleanWritable(b)
1724+
implicit def boolToBoolWritable (b: Boolean): BooleanWritable = new BooleanWritable(b)
17201725

1721-
implicit def bytesToBytesWritable (aob: Array[Byte]) = new BytesWritable(aob)
1726+
implicit def bytesToBytesWritable (aob: Array[Byte]): BytesWritable = new BytesWritable(aob)
17221727

1723-
implicit def stringToText(s: String) = new Text(s)
1728+
implicit def stringToText(s: String): Text = new Text(s)
17241729

17251730
private implicit def arrayToArrayWritable[T <% Writable: ClassTag](arr: Traversable[T])
17261731
: ArrayWritable = {

0 commit comments

Comments
 (0)