From c0648db2f924a544fd84b23783cf3ef448490915 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Tue, 4 Mar 2014 11:18:48 +0530 Subject: [PATCH 1/3] SPARK-1144 Added license and RAT to check licenses. removed rat source jar from the code base --- .rat-excludes | 36 ++++++++++++++++++++++++++++++++++++ dev/rat.bash | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ dev/run-tests | 15 +++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 .rat-excludes create mode 100755 dev/rat.bash diff --git a/.rat-excludes b/.rat-excludes new file mode 100644 index 000000000000..4d8c94140bf7 --- /dev/null +++ b/.rat-excludes @@ -0,0 +1,36 @@ +target +.gitignore +.project +.classpath +.rat-excludes +.*md +derby.log +TAGS +RELEASE +control +docs +fairscheduler.xml.template +log4j.properties +log4j.properties.template +metrics.properties.template +slaves +spark-env.sh +spark-env.sh.template +log4j-defaults.properties +sorttable.js +.*txt +.*data +.*log +cloudpickle.py +join.py +SparkExprTyper.scala +SparkILoop.scala +SparkILoopInit.scala +SparkIMain.scala +SparkImports.scala +SparkJLineCompletion.scala +SparkJLineReader.scala +SparkMemberHandlers.scala +sbt +sbt-launch-lib.bash +plugins.sbt diff --git a/dev/rat.bash b/dev/rat.bash new file mode 100755 index 000000000000..5d8d3f37353b --- /dev/null +++ b/dev/rat.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +acquire_rat_jar () { + + URL1="http://search.maven.org/remotecontent?filepath=org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" + URL2="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" + + JAR=$rat_jar + + if [[ ! -f "$rat_jar" ]]; then + # Download rat launch jar if it hasn't been downloaded yet + if [ ! -f ${JAR} ]; then + # Download + printf "Attempting to fetch rat\n" + JAR_DL=${JAR}.part + if hash curl 2>/dev/null; then + (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR} + elif hash wget 2>/dev/null; then + (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR} + else + printf "You do not have curl or wget installed, please install rat manually.\n" + exit -1 + fi + fi + if [ ! -f ${JAR} ]; then + # We failed to download + printf "Our attempt to download rat locally to ${JAR} failed. Please install rat manually.\n" + exit -1 + fi + printf "Launching rat from ${JAR}\n" + fi +} diff --git a/dev/run-tests b/dev/run-tests index cf0b940c09a8..98071e021e71 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -36,6 +36,21 @@ fi JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run, because JDK version is < 1.8." +export RAT_VERSION=0.10 +export rat_jar=$FWDIR/lib/apache-rat-${RAT_VERSION}.jar +mkdir -p $FWDIR/lib + +source $FWDIR/dev/rat.bash + +echo "=========================================================================" +echo "Running Apache RAT checks" +echo "=========================================================================" +[[ -f "$rat_jar" ]] || acquire_rat_jar || { + echo "Download failed. Obtain the rat jar manually and place it at $rat_jar" + exit 1 +} + +$java_cmd -jar $rat_jar -E $FWDIR/.rat-excludes -d $FWDIR echo "=========================================================================" echo "Running Scala style checks" From fcf28b183d01a6bb08c2e63b979d08f406b0d666 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Mon, 24 Mar 2014 12:34:32 +0530 Subject: [PATCH 2/3] Review feedback. --- .gitignore | 1 + .rat-excludes | 3 +++ dev/{rat.bash => check-license} | 31 +++++++++++++++++++++++++++++++ dev/run-tests | 14 ++------------ 4 files changed, 37 insertions(+), 12 deletions(-) rename dev/{rat.bash => check-license} (73%) diff --git a/.gitignore b/.gitignore index 3d178992123d..e9d8dc3147aa 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ dist/ spark-*-bin.tar.gz unit-tests.log /lib/ +rat-results.txt diff --git a/.rat-excludes b/.rat-excludes index 4d8c94140bf7..5c9915de9116 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -34,3 +34,6 @@ SparkMemberHandlers.scala sbt sbt-launch-lib.bash plugins.sbt +work +.*\.q +golden diff --git a/dev/rat.bash b/dev/check-license similarity index 73% rename from dev/rat.bash rename to dev/check-license index 5d8d3f37353b..9c6554e7e169 100755 --- a/dev/rat.bash +++ b/dev/check-license @@ -17,6 +17,7 @@ # limitations under the License. # + acquire_rat_jar () { URL1="http://search.maven.org/remotecontent?filepath=org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" @@ -47,3 +48,33 @@ acquire_rat_jar () { printf "Launching rat from ${JAR}\n" fi } + +# Go to the Spark project root directory +FWDIR="$(cd `dirname $0`/..; pwd)" +cd $FWDIR + +if test -x "$JAVA_HOME/bin/java"; then + declare java_cmd="$JAVA_HOME/bin/java" +else + declare java_cmd=java +fi + +export RAT_VERSION=0.10 +export rat_jar=$FWDIR/lib/apache-rat-${RAT_VERSION}.jar +mkdir -p $FWDIR/lib + +[[ -f "$rat_jar" ]] || acquire_rat_jar || { + echo "Download failed. Obtain the rat jar manually and place it at $rat_jar" + exit 1 +} + +$java_cmd -jar $rat_jar -E $FWDIR/.rat-excludes -d $FWDIR > rat-results.txt + +ERRORS=$(cat rat-results.txt | grep -e "??") + +if test ! -z "$ERRORS"; then + echo -e "Found errors: $ERRORS\n" + exit 1 +else + echo -e "RAT checks passed." +fi diff --git a/dev/run-tests b/dev/run-tests index 98071e021e71..6e78cad99500 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -34,23 +34,13 @@ else fi JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') -[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run, because JDK version is < 1.8." - -export RAT_VERSION=0.10 -export rat_jar=$FWDIR/lib/apache-rat-${RAT_VERSION}.jar -mkdir -p $FWDIR/lib - -source $FWDIR/dev/rat.bash +[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8." echo "=========================================================================" echo "Running Apache RAT checks" echo "=========================================================================" -[[ -f "$rat_jar" ]] || acquire_rat_jar || { - echo "Download failed. Obtain the rat jar manually and place it at $rat_jar" - exit 1 -} -$java_cmd -jar $rat_jar -E $FWDIR/.rat-excludes -d $FWDIR +dev/check-license echo "=========================================================================" echo "Running Scala style checks" From 64f7c7de368ef135bef2b6d8c44c19f6d3161b11 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Mon, 24 Mar 2014 12:53:32 +0530 Subject: [PATCH 3/3] added license headers. --- .../optimizer/FilterPushdownSuite.scala | 17 +++++++++++++++++ .../sql/catalyst/optimizer/OptimizerTest.scala | 17 +++++++++++++++++ .../spark/sql/columnar/ColumnTypeSuite.scala | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala index cfbef53de16b..0aa7e4d04d70 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.sql package catalyst package optimizer diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerTest.scala index 8ec1d3d8c03c..0c6e4d5acee8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/OptimizerTest.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.sql package catalyst package optimizer diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala index c7aaaae94ee5..1997d8638cb6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.spark.sql package columnar