From f7828040b39eb4f52a49f5c37e074d2803e5cc1f Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 10:56:56 -0700 Subject: [PATCH 01/10] Better script for QA automation --- dev/run-tests-jenkins | 71 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 dev/run-tests-jenkins diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins new file mode 100755 index 000000000000..fbde074c1392 --- /dev/null +++ b/dev/run-tests-jenkins @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Wrapper script that runs the Spark tests then reports QA results +# to github via its API. + +# Go to the Spark project root directory +FWDIR="$(cd `dirname $0`/..; pwd)" +cd $FWDIR + +# For backwards compatbility, just pass this through if we aren't on the +# Jenkins environment. +if [ -z "$AMPLAB_JENKINS" ]; then + ./dev/run-tests +fi + +github_message="QA results for PR $ghprbPullId:
" + +test_result=$(./dev/run-spark-tests) +if [ "$test_result" -eq "0" ]; then + github_message="$github_message- This patch PASSES unit tests.
" +else + github_message="$github_message- This patch FAILED unit tests.
" +fi + +if [ "$sha1" == "$ghprbActualCommit" ]; then + github_message="$github_message- This patch did not merge cleanly
" +else + github_message="$github_message- This patch merges cleanly
" + non_test_files=$(git diff master --name-only | grep -v "\/test" | tr "\n" " ") + new_public_classes=$(git diff $MASTER $non_test_files \ + | grep -e "trait " -e "class " \ + | grep "{" \ + | grep -v -e \@\@ -e private \ + | grep \+ \ + | sed "s/\+ *//" \ + | sed "s/$/
/") + if [ "$new_public_classes" == "" ]; then + github_message="$github_message- This patch adds no public classes
" + else + github_message="$github_message- This patch adds the following public classes (experimental):
" + github_message="
$github_message$new_public_classes" + fi +fi +github_message="
${github_message}For more information see $BUILD_URL" + +DEST_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" + +DATA="{\"body\": \"$github_message\"}" +echo "Attempting to post to Github:" +echo "$DATA" + +curl -D- -u x-oauth-basic:$GITHUB_OAUTH_KEY -X POST --data "$DATA" -H \ + "Content-Type: application/json" \ + $DEST_URL From a528a4b6aac4998ab264a22ba844db0fcc38fd44 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 12:04:58 -0700 Subject: [PATCH 02/10] Small fixes --- dev/run-tests-jenkins | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index fbde074c1392..2009171de5a1 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -24,15 +24,10 @@ FWDIR="$(cd `dirname $0`/..; pwd)" cd $FWDIR -# For backwards compatbility, just pass this through if we aren't on the -# Jenkins environment. -if [ -z "$AMPLAB_JENKINS" ]; then - ./dev/run-tests -fi - github_message="QA results for PR $ghprbPullId:
" -test_result=$(./dev/run-spark-tests) +./dev/run-tests +test_result="$?" if [ "$test_result" -eq "0" ]; then github_message="$github_message- This patch PASSES unit tests.
" else @@ -55,10 +50,10 @@ else github_message="$github_message- This patch adds no public classes
" else github_message="$github_message- This patch adds the following public classes (experimental):
" - github_message="
$github_message$new_public_classes" + github_message="$github_message
$new_public_classes" fi fi -github_message="
${github_message}For more information see $BUILD_URL" +github_message="${github_message}
For more information see $BUILD_URL" DEST_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" @@ -68,4 +63,6 @@ echo "$DATA" curl -D- -u x-oauth-basic:$GITHUB_OAUTH_KEY -X POST --data "$DATA" -H \ "Content-Type: application/json" \ - $DEST_URL + $DEST_URL | head -n 8 + +exit $test_result From 5f1ecb90dcd356ae4f0bd62214d5eea66a768f74 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 14:22:12 -0700 Subject: [PATCH 03/10] Add console link --- dev/run-tests-jenkins | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 2009171de5a1..2ea4dfb9ff0b 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -53,7 +53,8 @@ else github_message="$github_message
$new_public_classes" fi fi -github_message="${github_message}
For more information see $BUILD_URL" +github_message="${github_message}
For more information see test ouptut:" +github_message="${github_message}
${BUILD_URL}/consoleFull" DEST_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" From 4e3ca4541130d01f9298fa88a10b2fc86e1076f9 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 14:23:47 -0700 Subject: [PATCH 04/10] Adding test classes --- core/src/main/scala/org/apache/spark/SparkContext.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index 8bdaf0bf76e8..4398305f39b3 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -59,6 +59,10 @@ import org.apache.spark.util.{ClosureCleaner, MetadataCleaner, MetadataCleanerTy * this config overrides the default configs as well as system properties. */ +class PublicClass {} +class AnotherPublicClass {} +private[spark] class PrivateClass{} + class SparkContext(config: SparkConf) extends Logging { // This is used only by YARN for now, but should be relevant to other cluster types (Mesos, From 4f1094b4a7dda117bb34f38c827e24b2788a4fb7 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 15:39:20 -0700 Subject: [PATCH 05/10] Improvements for diff --- dev/run-tests-jenkins | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 2ea4dfb9ff0b..1d0531f01904 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -39,13 +39,14 @@ if [ "$sha1" == "$ghprbActualCommit" ]; then else github_message="$github_message- This patch merges cleanly
" non_test_files=$(git diff master --name-only | grep -v "\/test" | tr "\n" " ") - new_public_classes=$(git diff $MASTER $non_test_files \ + new_public_classes=$(git diff master $non_test_files \ | grep -e "trait " -e "class " \ | grep "{" \ | grep -v -e \@\@ -e private \ | grep \+ \ | sed "s/\+ *//" \ - | sed "s/$/
/") + | tr "\n" "~" \ + | sed "s/~/
/g") if [ "$new_public_classes" == "" ]; then github_message="$github_message- This patch adds no public classes
" else @@ -54,7 +55,7 @@ else fi fi github_message="${github_message}
For more information see test ouptut:" -github_message="${github_message}
${BUILD_URL}/consoleFull" +github_message="${github_message}
${BUILD_URL}consoleFull" DEST_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" From 536af4f43f773c71d40130bd8b87a995418908e8 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Thu, 12 Jun 2014 18:11:01 -0700 Subject: [PATCH 06/10] Minor changes --- dev/run-tests-jenkins | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 1d0531f01904..bed5af0b03b4 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -35,13 +35,13 @@ else fi if [ "$sha1" == "$ghprbActualCommit" ]; then - github_message="$github_message- This patch did not merge cleanly
" + github_message="$github_message- This patch DID NOT merge cleanly
" else github_message="$github_message- This patch merges cleanly
" non_test_files=$(git diff master --name-only | grep -v "\/test" | tr "\n" " ") new_public_classes=$(git diff master $non_test_files \ | grep -e "trait " -e "class " \ - | grep "{" \ + | grep -e "{" -e "(" \ | grep -v -e \@\@ -e private \ | grep \+ \ | sed "s/\+ *//" \ @@ -51,7 +51,7 @@ else github_message="$github_message- This patch adds no public classes
" else github_message="$github_message- This patch adds the following public classes (experimental):
" - github_message="$github_message
$new_public_classes" + github_message="$github_message$new_public_classes" fi fi github_message="${github_message}
For more information see test ouptut:" From 7ba86224246b059d9fbd00c77045d98ee1357d54 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Wed, 25 Jun 2014 13:16:02 -0700 Subject: [PATCH 07/10] Adding message when tests start --- dev/run-tests-jenkins | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index bed5af0b03b4..1707055bc2c0 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -24,20 +24,40 @@ FWDIR="$(cd `dirname $0`/..; pwd)" cd $FWDIR -github_message="QA results for PR $ghprbPullId:
" +COMMENTS_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" + +function post_message { + message=$1 + data="{\"body\": \"$message\"}" + echo "Attempting to post to Github:" + echo "$data" + + curl -D- -u x-oauth-basic:$GITHUB_OAUTH_KEY -X POST --data "$data" -H \ + "Content-Type: application/json" \ + $COMMENTS_URL | head -n 8 +} + +start_message="QA tests have started for PR $ghprbPullId:
" +start_message="${start_message}
For more information see test ouptut:" +start_message="${start_message}
${BUILD_URL}consoleFull" + +post_message "$start_message" ./dev/run-tests test_result="$?" + +result_message="QA results for PR $ghprbPullId:
" + if [ "$test_result" -eq "0" ]; then - github_message="$github_message- This patch PASSES unit tests.
" + result_message="$result_message- This patch PASSES unit tests.
" else - github_message="$github_message- This patch FAILED unit tests.
" + result_message="$result_message- This patch FAILED unit tests.
" fi if [ "$sha1" == "$ghprbActualCommit" ]; then - github_message="$github_message- This patch DID NOT merge cleanly
" + result_message="$result_message- This patch DID NOT merge cleanly
" else - github_message="$github_message- This patch merges cleanly
" + result_message="$result_message- This patch merges cleanly
" non_test_files=$(git diff master --name-only | grep -v "\/test" | tr "\n" " ") new_public_classes=$(git diff master $non_test_files \ | grep -e "trait " -e "class " \ @@ -48,23 +68,15 @@ else | tr "\n" "~" \ | sed "s/~/
/g") if [ "$new_public_classes" == "" ]; then - github_message="$github_message- This patch adds no public classes
" + result_message="$result_message- This patch adds no public classes
" else - github_message="$github_message- This patch adds the following public classes (experimental):
" - github_message="$github_message$new_public_classes" + result_message="$result_message- This patch adds the following public classes (experimental):
" + result_message="$result_message$new_public_classes" fi fi -github_message="${github_message}
For more information see test ouptut:" -github_message="${github_message}
${BUILD_URL}consoleFull" - -DEST_URL="https://api.github.com/repos/apache/spark/issues/$ghprbPullId/comments" - -DATA="{\"body\": \"$github_message\"}" -echo "Attempting to post to Github:" -echo "$DATA" +result_message="${result_message}
For more information see test ouptut:" +result_message="${result_message}
${BUILD_URL}consoleFull" -curl -D- -u x-oauth-basic:$GITHUB_OAUTH_KEY -X POST --data "$DATA" -H \ - "Content-Type: application/json" \ - $DEST_URL | head -n 8 +post_message "$result_message" exit $test_result From d8c9bced8454c526ba77a91ec3add96daf7f5f6e Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Wed, 25 Jun 2014 14:07:58 -0700 Subject: [PATCH 08/10] Small improvement --- dev/run-tests-jenkins | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 1707055bc2c0..967da82168ea 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -37,8 +37,7 @@ function post_message { $COMMENTS_URL | head -n 8 } -start_message="QA tests have started for PR $ghprbPullId:
" -start_message="${start_message}
For more information see test ouptut:" +start_message="QA tests have started for PR $ghprbPullId. For more information see test ouptut:" start_message="${start_message}
${BUILD_URL}consoleFull" post_message "$start_message" From 3e3ae0663124a9829f682270f17e52e04774fd08 Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Wed, 9 Jul 2014 10:50:30 +0900 Subject: [PATCH 09/10] More improvements --- dev/run-tests-jenkins | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 967da82168ea..5acd56f7004e 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -37,8 +37,14 @@ function post_message { $COMMENTS_URL | head -n 8 } -start_message="QA tests have started for PR $ghprbPullId. For more information see test ouptut:" -start_message="${start_message}
${BUILD_URL}consoleFull" +start_message="QA tests have started for PR $ghprbPullId." +if [ "$sha1" == "$ghprbActualCommit" ]; then + start_message="$start_message This patch DID NOT merge cleanly! " +else + start_message="$start_message This patch merges cleanly. " +fi +start_message="$start_message
To view progress see test ouptut:" +start_message="$start_message
${BUILD_URL}consoleFull" post_message "$start_message" @@ -53,9 +59,7 @@ else result_message="$result_message- This patch FAILED unit tests.
" fi -if [ "$sha1" == "$ghprbActualCommit" ]; then - result_message="$result_message- This patch DID NOT merge cleanly
" -else +if [ "$sha1" != "$ghprbActualCommit" ]; then result_message="$result_message- This patch merges cleanly
" non_test_files=$(git diff master --name-only | grep -v "\/test" | tr "\n" " ") new_public_classes=$(git diff master $non_test_files \ From 1f5f6d2338ee17da14601afd34ad5e313774c45f Mon Sep 17 00:00:00 2001 From: Patrick Wendell Date: Wed, 9 Jul 2014 10:55:37 +0900 Subject: [PATCH 10/10] Minor change --- dev/run-tests-jenkins | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/run-tests-jenkins b/dev/run-tests-jenkins index 5acd56f7004e..8dda671e976c 100755 --- a/dev/run-tests-jenkins +++ b/dev/run-tests-jenkins @@ -43,8 +43,8 @@ if [ "$sha1" == "$ghprbActualCommit" ]; then else start_message="$start_message This patch merges cleanly. " fi -start_message="$start_message
To view progress see test ouptut:" -start_message="$start_message
${BUILD_URL}consoleFull" +start_message="$start_message
View progress: " +start_message="$start_message${BUILD_URL}consoleFull" post_message "$start_message"