diff --git a/e2e/e2e-minikube.sh b/e2e/e2e-minikube.sh new file mode 100755 index 0000000..3d2aef2 --- /dev/null +++ b/e2e/e2e-minikube.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### This script can be used to run integration tests locally on minikube. +### Requirements: minikube v0.23+ with the DNS addon enabled, and kubectl configured to point to it. + +set -ex + +### Basic Validation ### +if [ ! -d "integration-test" ]; then + echo "This script must be invoked from the top-level directory of the integration-tests repository" + usage + exit 1 +fi + +# Set up config. +master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") +repo="https://github.com/apache/spark" +image_repo=test + +# Run tests in minikube mode. +./e2e/runner.sh -m $master -r $repo -i $image_repo -d minikube diff --git a/e2e/e2e-prow.sh b/e2e/e2e-prow.sh new file mode 100755 index 0000000..45a8c2b --- /dev/null +++ b/e2e/e2e-prow.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### This script is used by Kubernetes Test Infrastructure to run integration tests. +### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow +### To run the integration tests yourself, use e2e/runner.sh. + +set -ex + +# Install basic dependencies +echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +apt-get update && apt-get install -y curl wget git tar +apt-get install -t jessie-backports -y openjdk-8-jdk + +# Set up config. +master=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") +repo="https://github.com/apache/spark" + +# Special GCP project for publishing docker images built by test. +image_repo="gcr.io/spark-testing-191023" +cd "$(dirname "$0")"/../ +./e2e/runner.sh -m $master -r $repo -i $image_repo -d cloud + +# Copy out the junit xml files for consumption by k8s test-infra. +ls -1 ./integration-test/target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done diff --git a/e2e/runner.sh b/e2e/runner.sh new file mode 100755 index 0000000..8acecc2 --- /dev/null +++ b/e2e/runner.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +usage () { + echo "Usage:" + echo " ./e2e/runner.sh -h Display this help message." + echo " ./e2e/runner.sh -m -r -i -d [minikube|cloud]" + echo " note that you must have kubectl configured to access the specified" + echo " . Also you must have access to the . " + echo " The deployment mode can be specified using the 'd' flag." +} + +### Basic Validation ### +if [ ! -d "integration-test" ]; then + echo "This script must be invoked from the top-level directory of the integration-tests repository" + usage + exit 1 +fi + +### Set sensible defaults ### +REPO="https://github.com/apache/spark" +IMAGE_REPO="docker.io/kubespark" +DEPLOY_MODE="minikube" + +### Parse options ### +while getopts h:m:r:i:d: option +do + case "${option}" + in + h) + usage + exit 0 + ;; + m) MASTER=${OPTARG};; + r) REPO=${OPTARG};; + i) IMAGE_REPO=${OPTARG};; + d) DEPLOY_MODE=${OPTARG};; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + esac +done + +### Ensure cluster is set. +if [ -z "$MASTER" ] +then + echo "Missing master-url (-m) argument." + echo "" + usage + exit +fi + +### Ensure deployment mode is minikube/cloud. +if [[ $DEPLOY_MODE != minikube && $DEPLOY_MODE != cloud ]]; +then + echo "Invalid deployment mode $DEPLOY_MODE" + usage + exit 1 +fi + +echo "Running tests on cluster $MASTER against $REPO." +echo "Spark images will be created in $IMAGE_REPO" + +set -ex +root=$(pwd) + +# clone spark distribution if needed. +if [ -d "spark" ]; +then + (cd spark && git pull); +else + git clone $REPO; +fi + +cd spark && ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests +tag=$(git rev-parse HEAD | cut -c -6) +echo "Spark distribution built at SHA $tag" + +if [[ $DEPLOY_MODE == cloud ]] ; +then + cd dist && ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $tag build + if [[ $IMAGE_REPO == gcr.io* ]] ; + then + gcloud docker -- push $IMAGE_REPO/spark-driver:$tag && \ + gcloud docker -- push $IMAGE_REPO/spark-executor:$tag && \ + gcloud docker -- push $IMAGE_REPO/spark-init:$tag + else + ./sbin/build-push-docker-images.sh -r $IMAGE_REPO -t $tag push + fi +else + # -m option for minikube. + cd dist && ./sbin/build-push-docker-images.sh -m -r $IMAGE_REPO -t $tag build +fi + +cd $root/integration-test +$root/spark/build/mvn clean -Ddownload.plugin.skip=true integration-test \ + -Dspark-distro-tgz=$root/spark/*.tgz \ + -DextraScalaTestArgs="-Dspark.kubernetes.test.master=k8s://$MASTER \ + -Dspark.docker.test.driverImage=$IMAGE_REPO/spark-driver:$tag \ + -Dspark.docker.test.executorImage=$IMAGE_REPO/spark-executor:$tag" || : + +echo "TEST SUITE FINISHED"