Skip to content

Commit a1307ef

Browse files
pwendellpwendell
committed
Merge branch 'master' into branch-0.8
2 parents 86ca660 + 8c14f4b commit a1307ef

File tree

121 files changed

+2935
-619
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

121 files changed

+2935
-619
lines changed

‎LICENSE‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,3 +200,199 @@
200200
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201201
See the License for the specific language governing permissions and
202202
limitations under the License.
203+
204+
205+
=======================================================================
206+
Apache Spark Subcomponents:
207+
208+
The Apache Spark project contains subcomponents with separate copyright
209+
notices and license terms. Your use of the source code for the these
210+
subcomponents is subject to the terms and conditions of the following
211+
licenses.
212+
213+
214+
=======================================================================
215+
For the Boto EC2 library (ec2/third_party/boto*.zip):
216+
=======================================================================
217+
218+
Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/
219+
220+
Permission is hereby granted, free of charge, to any person obtaining a
221+
copy of this software and associated documentation files (the
222+
"Software"), to deal in the Software without restriction, including
223+
without limitation the rights to use, copy, modify, merge, publish, dis-
224+
tribute, sublicense, and/or sell copies of the Software, and to permit
225+
persons to whom the Software is furnished to do so, subject to the fol-
226+
lowing conditions:
227+
228+
The above copyright notice and this permission notice shall be included
229+
in all copies or substantial portions of the Software.
230+
231+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
232+
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
233+
ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
234+
SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
235+
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
236+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
237+
IN THE SOFTWARE.
238+
239+
240+
========================================================================
241+
For CloudPickle (pyspark/cloudpickle.py):
242+
========================================================================
243+
244+
Copyright (c) 2012, Regents of the University of California.
245+
Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
246+
All rights reserved.
247+
248+
Redistribution and use in source and binary forms, with or without
249+
modification, are permitted provided that the following conditions
250+
are met:
251+
* Redistributions of source code must retain the above copyright
252+
notice, this list of conditions and the following disclaimer.
253+
* Redistributions in binary form must reproduce the above copyright
254+
notice, this list of conditions and the following disclaimer in the
255+
documentation and/or other materials provided with the distribution.
256+
* Neither the name of the University of California, Berkeley nor the
257+
names of its contributors may be used to endorse or promote
258+
products derived from this software without specific prior written
259+
permission.
260+
261+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
262+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
263+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
264+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
265+
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
266+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
267+
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
268+
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
269+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
270+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
271+
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
272+
273+
274+
========================================================================
275+
For Py4J (python/lib/py4j0.7.egg and files in assembly/lib/net/sf/py4j):
276+
========================================================================
277+
278+
Copyright (c) 2009-2011, Barthelemy Dagenais All rights reserved.
279+
280+
Redistribution and use in source and binary forms, with or without
281+
modification, are permitted provided that the following conditions are met:
282+
283+
- Redistributions of source code must retain the above copyright notice, this
284+
list of conditions and the following disclaimer.
285+
286+
- Redistributions in binary form must reproduce the above copyright notice,
287+
this list of conditions and the following disclaimer in the documentation
288+
and/or other materials provided with the distribution.
289+
290+
- The name of the author may not be used to endorse or promote products
291+
derived from this software without specific prior written permission.
292+
293+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
294+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
295+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
296+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
297+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
298+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
299+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
300+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
301+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
302+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
303+
POSSIBILITY OF SUCH DAMAGE.
304+
305+
306+
========================================================================
307+
For DPark join code (python/pyspark/join.py):
308+
========================================================================
309+
310+
Copyright (c) 2011, Douban Inc. <http://www.douban.com/>
311+
All rights reserved.
312+
313+
Redistribution and use in source and binary forms, with or without
314+
modification, are permitted provided that the following conditions are
315+
met:
316+
317+
* Redistributions of source code must retain the above copyright
318+
notice, this list of conditions and the following disclaimer.
319+
320+
* Redistributions in binary form must reproduce the above
321+
copyright notice, this list of conditions and the following disclaimer
322+
in the documentation and/or other materials provided with the
323+
distribution.
324+
325+
* Neither the name of the Douban Inc. nor the names of its
326+
contributors may be used to endorse or promote products derived from
327+
this software without specific prior written permission.
328+
329+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
330+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
331+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
332+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
333+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
334+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
335+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
336+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
337+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
338+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
339+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
340+
341+
342+
========================================================================
343+
For sorttable (core/src/main/resources/org/apache/spark/ui/static/sorttable.js):
344+
========================================================================
345+
346+
Copyright (c) 1997-2007 Stuart Langridge
347+
348+
Permission is hereby granted, free of charge, to any person obtaining a copy
349+
of this software and associated documentation files (the "Software"), to deal
350+
in the Software without restriction, including without limitation the rights
351+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
352+
copies of the Software, and to permit persons to whom the Software is
353+
furnished to do so, subject to the following conditions:
354+
355+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
356+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
357+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
358+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
359+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
360+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
361+
THE SOFTWARE.
362+
363+
364+
========================================================================
365+
For Scala Interpreter classes (all .scala files in repl/src/main/scala
366+
except for Main.Scala, SparkHelper.scala and ExecutorClassLoader.scala):
367+
========================================================================
368+
369+
Copyright (c) 2002-2013 EPFL
370+
Copyright (c) 2011-2013 Typesafe, Inc.
371+
372+
All rights reserved.
373+
374+
Redistribution and use in source and binary forms, with or without
375+
modification, are permitted provided that the following conditions are met:
376+
377+
- Redistributions of source code must retain the above copyright notice,
378+
this list of conditions and the following disclaimer.
379+
380+
- Redistributions in binary form must reproduce the above copyright notice,
381+
this list of conditions and the following disclaimer in the documentation
382+
and/or other materials provided with the distribution.
383+
384+
- Neither the name of the EPFL nor the names of its contributors may be
385+
used to endorse or promote products derived from this software without
386+
specific prior written permission.
387+
388+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
389+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
390+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
391+
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
392+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
393+
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
394+
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
395+
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
396+
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
397+
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
398+
POSSIBILITY OF SUCH DAMAGE.

‎README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,17 @@ Please refer to the [Configuration guide](http://spark.incubator.apache.org/docs
8989
in the online documentation for an overview on how to configure Spark.
9090

9191

92+
## Apache Incubator Notice
93+
94+
Apache Spark is an effort undergoing incubation at The Apache Software
95+
Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of
96+
all newly accepted projects until a further review indicates that the
97+
infrastructure, communications, and decision making process have stabilized in
98+
a manner consistent with other successful ASF projects. While incubation status
99+
is not necessarily a reflection of the completeness or stability of the code,
100+
it does indicate that the project has yet to be fully endorsed by the ASF.
101+
102+
92103
## Contributing to Spark
93104

94105
Contributions via GitHub pull requests are gladly accepted from their original

‎assembly/README‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,9 @@ It creates a single tar.gz file that includes all needed dependency of the proje
44
except for org.apache.hadoop.* jars that are supposed to be available from the
55
deployed Hadoop cluster.
66

7-
This module is off by default to avoid spending extra time on top of repl-bin
8-
module. To activate it specify the profile in the command line
9-
-Passembly
7+
This module is off by default. To activate it specify the profile in the command line
8+
-Pbigtop-dist
109

11-
In case you want to avoid building time-expensive repl-bin module, that shaders
12-
all the dependency into a big flat jar supplement maven command with
13-
-DnoExpensive
10+
If you need to build an assembly for a different version of Hadoop the
11+
hadoop-version system property needs to be set as in this example:
12+
-Dhadoop.version=2.0.6-alpha

‎bin/compute-classpath.cmd‎

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -28,30 +28,27 @@ set FWDIR=%~dp0..\
2828
rem Load environment variables from conf\spark-env.cmd, if it exists
2929
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd"
3030

31-
set CORE_DIR=%FWDIR%core
32-
set REPL_DIR=%FWDIR%repl
33-
set EXAMPLES_DIR=%FWDIR%examples
34-
set BAGEL_DIR=%FWDIR%bagel
35-
set MLLIB_DIR=%FWDIR%mllib
36-
set TOOLS_DIR=%FWDIR%tools
37-
set YARN_DIR=%FWDIR%yarn
38-
set STREAMING_DIR=%FWDIR%streaming
39-
set PYSPARK_DIR=%FWDIR%python
40-
4131
rem Build up classpath
42-
set CLASSPATH=%SPARK_CLASSPATH%;%MESOS_CLASSPATH%;%FWDIR%conf;%CORE_DIR%\target\scala-%SCALA_VERSION%\classes
43-
set CLASSPATH=%CLASSPATH%;%CORE_DIR%\target\scala-%SCALA_VERSION%\test-classes;%CORE_DIR%\src\main\resources
44-
set CLASSPATH=%CLASSPATH%;%STREAMING_DIR%\target\scala-%SCALA_VERSION%\classes;%STREAMING_DIR%\target\scala-%SCALA_VERSION%\test-classes
45-
set CLASSPATH=%CLASSPATH%;%STREAMING_DIR%\lib\org\apache\kafka\kafka\0.7.2-spark\*
46-
set CLASSPATH=%CLASSPATH%;%REPL_DIR%\target\scala-%SCALA_VERSION%\classes;%EXAMPLES_DIR%\target\scala-%SCALA_VERSION%\classes
47-
set CLASSPATH=%CLASSPATH%;%FWDIR%lib_managed\jars\*
48-
set CLASSPATH=%CLASSPATH%;%FWDIR%lib_managed\bundles\*
49-
set CLASSPATH=%CLASSPATH%;%FWDIR%repl\lib\*
50-
set CLASSPATH=%CLASSPATH%;%FWDIR%python\lib\*
51-
set CLASSPATH=%CLASSPATH%;%BAGEL_DIR%\target\scala-%SCALA_VERSION%\classes
52-
set CLASSPATH=%CLASSPATH%;%MLLIB_DIR%\target\scala-%SCALA_VERSION%\classes
53-
set CLASSPATH=%CLASSPATH%;%TOOLS_DIR%\target\scala-%SCALA_VERSION%\classes
54-
set CLASSPATH=%CLASSPATH%;%YARN_DIR%\target\scala-%SCALA_VERSION%\classes
32+
set CLASSPATH=%SPARK_CLASSPATH%;%FWDIR%conf
33+
if exist "%FWDIR%RELEASE" (
34+
for %%d in ("%FWDIR%jars\spark-assembly*.jar") do (
35+
set ASSEMBLY_JAR=%%d
36+
)
37+
) else (
38+
for %%d in ("%FWDIR%assembly\target\scala-%SCALA_VERSION%\spark-assembly*hadoop*.jar") do (
39+
set ASSEMBLY_JAR=%%d
40+
)
41+
)
42+
set CLASSPATH=%CLASSPATH%;%ASSEMBLY_JAR%
43+
44+
if "x%SPARK_TESTING%"=="x1" (
45+
rem Add test clases to path
46+
set CLASSPATH=%CLASSPATH%;%FWDIR%core\target\scala-%SCALA_VERSION%\test-classes
47+
set CLASSPATH=%CLASSPATH%;%FWDIR%repl\target\scala-%SCALA_VERSION%\test-classes
48+
set CLASSPATH=%CLASSPATH%;%FWDIR%mllib\target\scala-%SCALA_VERSION%\test-classes
49+
set CLASSPATH=%CLASSPATH%;%FWDIR%bagel\target\scala-%SCALA_VERSION%\test-classes
50+
set CLASSPATH=%CLASSPATH%;%FWDIR%streaming\target\scala-%SCALA_VERSION%\test-classes
51+
)
5552

5653
rem Add hadoop conf dir - else FileSystem.*, etc fail
5754
rem Note, this assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
@@ -64,9 +61,6 @@ if "x%YARN_CONF_DIR%"=="x" goto no_yarn_conf_dir
6461
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%
6562
:no_yarn_conf_dir
6663

67-
rem Add Scala standard library
68-
set CLASSPATH=%CLASSPATH%;%SCALA_HOME%\lib\scala-library.jar;%SCALA_HOME%\lib\scala-compiler.jar;%SCALA_HOME%\lib\jline.jar
69-
7064
rem A bit of a hack to allow calling this script within run2.cmd without seeing output
7165
if "%DONT_PRINT_CLASSPATH%"=="1" goto exit
7266

‎conf/fairscheduler.xml.template‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
<?xml version="1.0"?>
22
<allocations>
3-
<pool name="production">
4-
<minShare>2</minShare>
5-
<weight>1</weight>
3+
<pool name="production">
64
<schedulingMode>FAIR</schedulingMode>
7-
</pool>
8-
<pool name="test">
9-
<minShare>3</minShare>
10-
<weight>2</weight>
5+
<weight>1</weight>
6+
<minShare>2</minShare>
7+
</pool>
8+
<pool name="test">
119
<schedulingMode>FIFO</schedulingMode>
12-
</pool>
13-
<pool name="data">
14-
</pool>
10+
<weight>2</weight>
11+
<minShare>3</minShare>
12+
</pool>
1513
</allocations>

0 commit comments

Comments
 (0)