From 2c5305128eb5d2e7847a0b6df67b7068d7f0ad8f Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 28 Feb 2018 12:10:46 -0800 Subject: [PATCH 01/39] Initial checkin of k8s integration tests. These tests were developed in the https://github.com/apache-spark-on-k8s/spark-integration repo by several contributors. This is a copy of the current state into the main apache spark repo. The only changes from the current spark-integration repo state are: * Move the files from the repo root into resource-managers/kubernetes/integration-tests * Add a reference to these tests in the root README.md * Fix a path reference in dev/dev-run-integration-tests.sh * Add a TODO in include/util.sh --- README.md | 2 + .../kubernetes/integration-tests/LICENSE | 201 +++++++++ .../kubernetes/integration-tests/README.md | 82 ++++ .../kubernetes/integration-tests/build/mvn | 29 ++ .../dev/dev-run-integration-tests.sh | 110 +++++ .../integration-tests/dev/spark-rbac.yaml | 52 +++ .../integration-tests/e2e/e2e-prow.sh | 69 ++++ .../integration-tests/include/util.sh | 43 ++ .../kubernetes/integration-tests/pom.xml | 212 ++++++++++ .../scripts/setup-integration-test-env.sh | 91 ++++ .../src/test/resources/log4j.properties | 31 ++ .../k8s/integrationtest/KubernetesSuite.scala | 391 ++++++++++++++++++ .../KubernetesTestComponents.scala | 115 ++++++ .../deploy/k8s/integrationtest/Logging.scala | 35 ++ .../k8s/integrationtest/ProcessUtils.scala | 44 ++ .../SparkReadinessWatcher.scala | 41 ++ .../deploy/k8s/integrationtest/Utils.scala | 86 ++++ .../backend/IntegrationTestBackend.scala | 41 ++ .../backend/cloud/CloudTestBackend.scala | 41 ++ .../backend/minikube/Minikube.scala | 83 ++++ .../minikube/MinikubeTestBackend.scala | 42 ++ .../deploy/k8s/integrationtest/config.scala | 38 ++ .../k8s/integrationtest/constants.scala | 22 + 23 files changed, 1901 insertions(+) create mode 100644 resource-managers/kubernetes/integration-tests/LICENSE create mode 100644 resource-managers/kubernetes/integration-tests/README.md create mode 100755 resource-managers/kubernetes/integration-tests/build/mvn create mode 100755 resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh create mode 100644 resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml create mode 100755 resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh create mode 100644 resource-managers/kubernetes/integration-tests/include/util.sh create mode 100644 resource-managers/kubernetes/integration-tests/pom.xml create mode 100755 resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh create mode 100644 resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala diff --git a/README.md b/README.md index 1e521a7e7b178..531d330234062 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ can be run using: Please see the guidance on how to [run tests for a module, or individual tests](http://spark.apache.org/developer-tools.html#individual-tests). +There is also a Kubernetes integration test, see resource-managers/kubernetes/integration-tests/README.md + ## A Note About Hadoop Versions Spark uses the Hadoop core library to talk to HDFS and other Hadoop-supported diff --git a/resource-managers/kubernetes/integration-tests/LICENSE b/resource-managers/kubernetes/integration-tests/LICENSE new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md new file mode 100644 index 0000000000000..97cb8ab42e7a0 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -0,0 +1,82 @@ +--- +layout: global +title: Spark on Kubernetes Integration Tests +--- + +# Running the Kubernetes Integration Tests + +Note that the integration test framework is currently being heavily revised and +is subject to change. Note that currently the integration tests only run with Java 8. + +The simplest way to run the integration tests is to install and run Minikube, then run the following: + + dev/dev-run-integration-tests.sh + +The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should +run with a minimum of 3 CPUs and 4G of memory: + + minikube start --cpus 3 --memory 4096 + +You can download Minikube [here](https://github.com/kubernetes/minikube/releases). + +# Integration test customization + +Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. + +## Use a non-local cluster + +To use your own cluster running in the cloud, set the following: + +* `--deploy-mode cloud` to indicate that the test is connecting to a remote cluster instead of Minikube, +* `--spark-master ` - set `` to the externally accessible Kubernetes cluster URL, +* `--image-repo ` - set `` to a write-accessible Docker image repository that provides the images for your cluster. The framework assumes your local Docker client can push to this repository. + +Therefore the command looks like this: + + dev/dev-run-integration-tests.sh \ + --deploy-mode cloud \ + --spark-master https://example.com:8443/apiserver \ + --image-repo docker.example.com/spark-images + +## Re-using Docker Images + +By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, +and it is written to file at `target/imageTag.txt`. To reuse the images built in a previous run, or to use a Docker image tag +that you have built by other means already, pass the tag to the test script: + + dev/dev-run-integration-tests.sh --image-tag + +where if you still want to use images that were built before by the test framework: + + dev/dev-run-integration-tests.sh --image-tag $(cat target/imageTag.txt) + +## Customizing the Spark Source Code to Test + +By default, the test framework will test the master branch of Spark from [here](https://github.com/apache/spark). You +can specify the following options to test against different source versions of Spark: + +* `--spark-repo ` - set `` to the git or http URI of the Spark git repository to clone +* `--spark-branch ` - set `` to the branch of the repository to build. + + +An example: + + dev/dev-run-integration-tests.sh \ + --spark-repo https://github.com/apache-spark-on-k8s/spark \ + --spark-branch new-feature + +Additionally, you can use a pre-built Spark distribution. In this case, the repository is not cloned at all, and no +source code has to be compiled. + +* `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. + +When the tests are cloning a repository and building it, the Spark distribution is placed in `target/spark/spark-.tgz`. +Reuse this tarball to save a significant amount of time if you are iterating on the development of these integration tests. + +## Customizing the Namespace and Service Account + +* `--namespace ` - set `` to the namespace in which the tests should be run. +* `--service-account ` - set `` to the name of the Kubernetes service account to +use in the namespace specified by the `--namespace`. The service account is expected to have permissions to get, list, watch, +and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account +in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/build/mvn b/resource-managers/kubernetes/integration-tests/build/mvn new file mode 100755 index 0000000000000..87e5b58640c5c --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/build/mvn @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BUILD_DIR=$(dirname $0) + +MVN_RUNNER=$BUILD_DIR/run-mvn + +if [ ! -f $MVN_RUNNER ]; +then + curl -s --progress-bar https://raw.githubusercontent.com/apache/spark/master/build/mvn > $MVN_RUNNER + chmod +x $MVN_RUNNER +fi +source $MVN_RUNNER diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh new file mode 100755 index 0000000000000..33879d5fc8b2a --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +source ./include/util.sh + +TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests +BRANCH="master" +SPARK_REPO="https://github.com/apache/spark" +SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +SPARK_TGZ="N/A" +IMAGE_TAG="N/A" +SPARK_MASTER= +NAMESPACE= +SERVICE_ACCOUNT= + +# Parse arguments +while (( "$#" )); do + case $1 in + --spark-branch) + BRANCH="$2" + shift + ;; + --spark-repo) + SPARK_REPO="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + --spark-master) + SPARK_MASTER="$2" + shift + ;; + --namespace) + NAMESPACE="$2" + shift + ;; + --service-account) + SERVICE_ACCOUNT="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." + clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH +fi + +cd $TEST_ROOT_DIR + +properties=( + -Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \ + -Dspark.kubernetes.test.imageTag=$IMAGE_TAG \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE +) + +if [ -n $NAMESPACE ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE ) +fi + +if [ -n $SERVICE_ACCOUNT ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT ) +fi + +if [ -n $SPARK_MASTER ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER ) +fi + +build/mvn integration-test ${properties[@]} diff --git a/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml b/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml new file mode 100644 index 0000000000000..a4c242f2f2645 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Namespace +metadata: + name: spark +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark-sa + namespace: spark +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: spark-role +rules: +- apiGroups: + - "" + resources: + - "pods" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: spark-role-binding +subjects: +- kind: ServiceAccount + name: spark-sa + namespace: spark +roleRef: + kind: ClusterRole + name: spark-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh new file mode 100755 index 0000000000000..501f350943e73 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### This script is used by Kubernetes Test Infrastructure to run integration tests. +### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow + +set -ex + +# set cwd correctly +cd "$(dirname "$0")/../" + +# Include requisite scripts +source ./include/util.sh + +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +BRANCH="master" +SPARK_REPO="https://github.com/apache/spark" +SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" + +## Install basic dependencies +## These are for the kubekins-e2e environment in https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e +echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +apt-get update && apt-get install -y curl wget git tar uuid-runtime +apt-get install -t jessie-backports -y openjdk-8-jdk + +# Set up config. +MASTER=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") + +# Special GCP project for publishing docker images built by test. +IMAGE_REPO="gcr.io/spark-testing-191023" + +# Cloning the spark distribution +echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." +clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH + +# Spark distribution +properties=( + -Dspark.kubernetes.test.master=k8s://$MASTER \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.sparkTgz="$SPARK_TGZ" \ + -Dspark.kubernetes.test.deployMode=cloud \ + -Dspark.kubernetes.test.namespace=spark \ + -Dspark.kubernetes.test.serviceAccountName=spark-sa +) + +# Run kubectl commands and create appropriate roles +kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com +kubectl create -f ./dev/spark-rbac.yaml + +# Run tests. +echo "Starting test with ${properties[@]}" +build/mvn integration-test "${properties[@]}" || : + +# Copy out the junit xml files for consumption by k8s test-infra. +ls -1 ./target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done diff --git a/resource-managers/kubernetes/integration-tests/include/util.sh b/resource-managers/kubernetes/integration-tests/include/util.sh new file mode 100644 index 0000000000000..49b9114854120 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/include/util.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +clone_build_spark() { + spark_repo=$1 + spark_repo_local_dir=$2 + branch=$3 + pushd . + + # clone spark distribution if needed. + # TODO(ssuchter): This code, that does a checkout of a spark repo, + # made more sense when this script was in the repo + # https://github.com/apache-spark-on-k8s/spark-integration + # but now we shouldn't check out another copy of spark, we should just + # build in the copy that is checked out already. + if [ -d "$spark_repo_local_dir" ]; + then + (cd $spark_repo_local_dir && git fetch origin $branch); + else + mkdir -p $spark_repo_local_dir; + git clone -b $branch --single-branch $spark_repo $spark_repo_local_dir; + fi + cd $spark_repo_local_dir + git checkout -B $branch origin/$branch + ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests; + SPARK_TGZ=$(find $spark_repo_local_dir -name spark-*.tgz) + + popd +} diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml new file mode 100644 index 0000000000000..e3e48fcc07f7d --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -0,0 +1,212 @@ + + + + 4.0.0 + + spark-kubernetes-integration-tests_2.11 + spark-kubernetes-integration-tests + 0.1-SNAPSHOT + + 3.3.9 + 3.5 + 1.1.1 + 5.0.2 + 1.3.0 + 1.4.0 + + 18.0 + 1.3.9 + 3.0.0 + 1.2.17 + 2.11.8 + 2.11 + 3.2.2 + 2.2.6 + 1.0 + 1.7.24 + kubernetes-integration-tests + ${project.build.directory}/spark-dist-unpacked + N/A + ${project.build.directory}/imageTag.txt + minikube + docker.io/kubespark + + + jar + Spark Project Kubernetes Integration Tests + + + + commons-logging + commons-logging + ${commons-logging.version} + + + com.google.code.findbugs + jsr305 + ${jsr305.version} + + + com.google.guava + guava + test + + ${guava.version} + + + com.spotify + docker-client + ${docker-client.version} + test + + + io.fabric8 + kubernetes-client + ${kubernetes-client.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + org.scala-lang + scala-library + ${scala.version} + + + org.scalatest + scalatest_${scala.binary.version} + ${scalatest.version} + test + + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + test + + + + + + + net.alchim31.maven + scala-maven-plugin + ${scala-maven-plugin.version} + + + + compile + testCompile + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + setup-integration-test-env + pre-integration-test + + exec + + + scripts/setup-integration-test-env.sh + + --unpacked-spark-tgz + ${spark.kubernetes.test.unpackSparkDir} + + --image-repo + ${spark.kubernetes.test.imageRepo} + + --image-tag + ${spark.kubernetes.test.imageTag} + + --image-tag-output-file + ${spark.kubernetes.test.imageTagFile} + + --deploy-mode + ${spark.kubernetes.test.deployMode} + + --spark-tgz + ${spark.kubernetes.test.sparkTgz} + + + + + + + + org.scalatest + scalatest-maven-plugin + ${scalatest-maven-plugin.version} + + ${project.build.directory}/surefire-reports + . + SparkTestSuite.txt + -ea -Xmx3g -XX:ReservedCodeCacheSize=512m ${extraScalaTestArgs} + + + file:src/test/resources/log4j.properties + true + ${spark.kubernetes.test.imageTagFile} + ${spark.kubernetes.test.unpackSparkDir} + ${spark.kubernetes.test.imageRepo} + ${spark.kubernetes.test.deployMode} + ${spark.kubernetes.test.master} + ${spark.kubernetes.test.namespace} + ${spark.kubernetes.test.serviceAccountName} + + ${test.exclude.tags} + + + + test + + test + + + + (?<!Suite) + + + + integration-test + integration-test + + test + + + + + + + + + diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh new file mode 100755 index 0000000000000..ccfb8e767c529 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +UNPACKED_SPARK_TGZ="$TEST_ROOT_DIR/target/spark-dist-unpacked" +IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +IMAGE_TAG="N/A" +SPARK_TGZ="N/A" + +# Parse arguments +while (( "$#" )); do + case $1 in + --unpacked-spark-tgz) + UNPACKED_SPARK_TGZ="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --image-tag-output-file) + IMAGE_TAG_OUTPUT_FILE="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Must specify a Spark tarball to build Docker images against with --spark-tgz." && exit 1; +fi + +rm -rf $UNPACKED_SPARK_TGZ +mkdir -p $UNPACKED_SPARK_TGZ +tar -xzvf $SPARK_TGZ --strip-components=1 -C $UNPACKED_SPARK_TGZ; + +if [[ $IMAGE_TAG == "N/A" ]]; +then + IMAGE_TAG=$(uuidgen); + cd $UNPACKED_SPARK_TGZ + if [[ $DEPLOY_MODE == cloud ]] ; + then + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build + if [[ $IMAGE_REPO == gcr.io* ]] ; + then + gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG + else + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push + fi + else + # -m option for minikube. + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build + fi + cd - +fi + +rm -f $IMAGE_TAG_OUTPUT_FILE +echo -n $IMAGE_TAG > $IMAGE_TAG_OUTPUT_FILE diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties b/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..866126bc3c1c2 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/integration-tests.log +log4j.rootCategory=INFO, file +log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file.append=true +log4j.appender.file.file=target/integration-tests.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n + +# Ignore messages below warning level from a few verbose libraries. +log4j.logger.com.sun.jersey=WARN +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.mortbay=WARN +log4j.logger.org.spark_project.jetty=WARN diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala new file mode 100644 index 0000000000000..89c8a91c4ab58 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.File +import java.nio.file.{Path, Paths} +import java.util.UUID +import java.util.regex.Pattern + +import scala.collection.JavaConverters._ +import com.google.common.io.PatternFilenameFilter +import io.fabric8.kubernetes.api.model.{Container, Pod} +import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite} +import org.scalatest.concurrent.{Eventually, PatienceConfiguration} +import org.scalatest.time.{Minutes, Seconds, Span} + +import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} +import org.apache.spark.deploy.k8s.integrationtest.config._ + +private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll with BeforeAndAfter { + + import KubernetesSuite._ + + private var testBackend: IntegrationTestBackend = _ + private var sparkHomeDir: Path = _ + private var kubernetesTestComponents: KubernetesTestComponents = _ + private var sparkAppConf: SparkAppConf = _ + private var image: String = _ + private var containerLocalSparkDistroExamplesJar: String = _ + private var appLocator: String = _ + private var driverPodName: String = _ + + override def beforeAll(): Unit = { + // The scalatest-maven-plugin gives system properties that are referenced but not set null + // values. We need to remove the null-value properties before initializing the test backend. + val nullValueProperties = System.getProperties.asScala + .filter(entry => entry._2.equals("null")) + .map(entry => entry._1.toString) + nullValueProperties.foreach { key => + System.clearProperty(key) + } + + val sparkDirProp = System.getProperty("spark.kubernetes.test.unpackSparkDir") + require(sparkDirProp != null, "Spark home directory must be provided in system properties.") + sparkHomeDir = Paths.get(sparkDirProp) + require(sparkHomeDir.toFile.isDirectory, + s"No directory found for spark home specified at $sparkHomeDir.") + val imageTag = getTestImageTag + val imageRepo = getTestImageRepo + image = s"$imageRepo/spark:$imageTag" + + val sparkDistroExamplesJarFile: File = sparkHomeDir.resolve(Paths.get("examples", "jars")) + .toFile + .listFiles(new PatternFilenameFilter(Pattern.compile("^spark-examples_.*\\.jar$")))(0) + containerLocalSparkDistroExamplesJar = s"local:///opt/spark/examples/jars/" + + s"${sparkDistroExamplesJarFile.getName}" + testBackend = IntegrationTestBackendFactory.getTestBackend + testBackend.initialize() + kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient) + } + + override def afterAll(): Unit = { + testBackend.cleanUp() + } + + before { + appLocator = UUID.randomUUID().toString.replaceAll("-", "") + driverPodName = "spark-test-app-" + UUID.randomUUID().toString.replaceAll("-", "") + sparkAppConf = kubernetesTestComponents.newSparkAppConf() + .set("spark.kubernetes.container.image", image) + .set("spark.kubernetes.driver.pod.name", driverPodName) + .set("spark.kubernetes.driver.label.spark-app-locator", appLocator) + .set("spark.kubernetes.executor.label.spark-app-locator", appLocator) + if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { + kubernetesTestComponents.createNamespace() + } + } + + after { + if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { + kubernetesTestComponents.deleteNamespace() + } + deleteDriverPod() + } + + test("Run SparkPi with no resources") { + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with a very long application name.") { + sparkAppConf.set("spark.app.name", "long" * 40) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with a master URL without a scheme.") { + val url = kubernetesTestComponents.kubernetesClient.getMasterUrl + val k8sMasterUrl = if (url.getPort < 0) { + s"k8s://${url.getHost}" + } else { + s"k8s://${url.getHost}:${url.getPort}" + } + sparkAppConf.set("spark.master", k8sMasterUrl) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with an argument.") { + runSparkPiAndVerifyCompletion(appArgs = Array("5")) + } + + test("Run SparkPi with custom labels, annotations, and environment variables.") { + sparkAppConf + .set("spark.kubernetes.driver.label.label1", "label1-value") + .set("spark.kubernetes.driver.label.label2", "label2-value") + .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") + .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") + .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") + .set("spark.kubernetes.executor.label.label1", "label1-value") + .set("spark.kubernetes.executor.label.label2", "label2-value") + .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") + .set("spark.executorEnv.ENV1", "VALUE1") + .set("spark.executorEnv.ENV2", "VALUE2") + + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkCustomSettings(driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkCustomSettings(executorPod) + }) + } + + test("Run SparkPi with a test secret mounted into the driver and executor pods") { + val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") + createTestSecret(secretName) + + sparkAppConf + .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + + try { + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkTestSecret(secretName, driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkTestSecret(secretName, executorPod) + }) + } finally { + deleteTestSecret(secretName) + } + } + + test("Run PageRank using remote data file") { + sparkAppConf + .set("spark.kubernetes.mountDependencies.filesDownloadDir", + CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + runSparkPageRankAndVerifyCompletion( + appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) + } + + test("Run PageRank using remote data file with test secret mounted into the driver and " + + "executors") { + val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") + createTestSecret(secretName) + + sparkAppConf + .set("spark.kubernetes.mountDependencies.filesDownloadDir", + CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + + try { + runSparkPageRankAndVerifyCompletion( + appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkTestSecret(secretName, driverPod, withInitContainer = true) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkTestSecret(secretName, executorPod, withInitContainer = true) + }) + } finally { + deleteTestSecret(secretName) + } + } + + private def runSparkPiAndVerifyCompletion( + appResource: String = containerLocalSparkDistroExamplesJar, + driverPodChecker: Pod => Unit = doBasicDriverPodCheck, + executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, + appArgs: Array[String] = Array.empty[String], + appLocator: String = appLocator): Unit = { + runSparkApplicationAndVerifyCompletion( + appResource, + SPARK_PI_MAIN_CLASS, + Seq("Pi is roughly 3"), + appArgs, + driverPodChecker, + executorPodChecker, + appLocator) + } + + private def runSparkPageRankAndVerifyCompletion( + appResource: String = containerLocalSparkDistroExamplesJar, + driverPodChecker: Pod => Unit = doBasicDriverPodCheck, + executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, + appArgs: Array[String], + appLocator: String = appLocator): Unit = { + runSparkApplicationAndVerifyCompletion( + appResource, + SPARK_PAGE_RANK_MAIN_CLASS, + Seq("1 has rank", "2 has rank", "3 has rank", "4 has rank"), + appArgs, + driverPodChecker, + executorPodChecker, + appLocator) + } + + private def runSparkApplicationAndVerifyCompletion( + appResource: String, + mainClass: String, + expectedLogOnCompletion: Seq[String], + appArgs: Array[String], + driverPodChecker: Pod => Unit, + executorPodChecker: Pod => Unit, + appLocator: String): Unit = { + val appArguments = SparkAppArguments( + mainAppResource = appResource, + mainClass = mainClass, + appArgs = appArgs) + SparkAppLauncher.launch(appArguments, sparkAppConf, TIMEOUT.value.toSeconds.toInt, sparkHomeDir) + + val driverPod = kubernetesTestComponents.kubernetesClient + .pods() + .withLabel("spark-app-locator", appLocator) + .withLabel("spark-role", "driver") + .list() + .getItems + .get(0) + driverPodChecker(driverPod) + + val executorPods = kubernetesTestComponents.kubernetesClient + .pods() + .withLabel("spark-app-locator", appLocator) + .withLabel("spark-role", "executor") + .list() + .getItems + executorPods.asScala.foreach { pod => + executorPodChecker(pod) + } + + Eventually.eventually(TIMEOUT, INTERVAL) { + expectedLogOnCompletion.foreach { e => + assert(kubernetesTestComponents.kubernetesClient + .pods() + .withName(driverPod.getMetadata.getName) + .getLog + .contains(e), "The application did not complete.") + } + } + } + + private def doBasicDriverPodCheck(driverPod: Pod): Unit = { + assert(driverPod.getMetadata.getName === driverPodName) + assert(driverPod.getSpec.getContainers.get(0).getImage === image) + assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver") + } + + private def doBasicExecutorPodCheck(executorPod: Pod): Unit = { + assert(executorPod.getSpec.getContainers.get(0).getImage === image) + assert(executorPod.getSpec.getContainers.get(0).getName === "executor") + } + + private def checkCustomSettings(pod: Pod): Unit = { + assert(pod.getMetadata.getLabels.get("label1") === "label1-value") + assert(pod.getMetadata.getLabels.get("label2") === "label2-value") + assert(pod.getMetadata.getAnnotations.get("annotation1") === "annotation1-value") + assert(pod.getMetadata.getAnnotations.get("annotation2") === "annotation2-value") + + val container = pod.getSpec.getContainers.get(0) + val envVars = container + .getEnv + .asScala + .map { env => + (env.getName, env.getValue) + } + .toMap + assert(envVars("ENV1") === "VALUE1") + assert(envVars("ENV2") === "VALUE2") + } + + private def deleteDriverPod(): Unit = { + kubernetesTestComponents.kubernetesClient.pods().withName(driverPodName).delete() + Eventually.eventually(TIMEOUT, INTERVAL) { + assert(kubernetesTestComponents.kubernetesClient + .pods() + .withName(driverPodName) + .get() == null) + } + } + + private def createTestSecret(secretName: String): Unit = { + kubernetesTestComponents.kubernetesClient.secrets + .createNew() + .editOrNewMetadata() + .withName(secretName) + .endMetadata() + .addToStringData(TEST_SECRET_KEY, TEST_SECRET_VALUE) + .done() + } + + private def checkTestSecret( + secretName: String, + pod: Pod, + withInitContainer: Boolean = false): Unit = { + val testSecretVolume = pod.getSpec.getVolumes.asScala.filter { volume => + volume.getName == s"$secretName-volume" + } + assert(testSecretVolume.size === 1) + assert(testSecretVolume.head.getSecret.getSecretName === secretName) + + checkTestSecretInContainer(secretName, pod.getSpec.getContainers.get(0)) + + if (withInitContainer) { + checkTestSecretInContainer(secretName, pod.getSpec.getInitContainers.get(0)) + } + } + + private def checkTestSecretInContainer(secretName: String, container: Container): Unit = { + val testSecret = container.getVolumeMounts.asScala.filter { mount => + mount.getName == s"$secretName-volume" + } + assert(testSecret.size === 1) + assert(testSecret.head.getMountPath === TEST_SECRET_MOUNT_PATH) + } + + private def deleteTestSecret(secretName: String): Unit = { + kubernetesTestComponents.kubernetesClient.secrets + .withName(secretName) + .delete() + + Eventually.eventually(TIMEOUT, INTERVAL) { + assert(kubernetesTestComponents.kubernetesClient.secrets.withName(secretName).get() == null) + } + } +} + +private[spark] object KubernetesSuite { + + val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes)) + val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds)) + val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" + val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" + + val TEST_SECRET_NAME_PREFIX = "test-secret-" + val TEST_SECRET_KEY = "test-key" + val TEST_SECRET_VALUE = "test-data" + val TEST_SECRET_MOUNT_PATH = "/etc/secrets" + + val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" + + val REMOTE_PAGE_RANK_DATA_FILE = + "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" + val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = + s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" + + case object ShuffleNotReadyException extends Exception +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala new file mode 100644 index 0000000000000..b9c87d59fe27b --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.nio.file.{Path, Paths} +import java.util.UUID + +import scala.collection.mutable +import scala.collection.JavaConverters._ +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import org.scalatest.concurrent.Eventually + +private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) { + + val namespaceOption = Option(System.getProperty("spark.kubernetes.test.namespace")) + val hasUserSpecifiedNamespace = namespaceOption.isDefined + val namespace = namespaceOption.getOrElse(UUID.randomUUID().toString.replaceAll("-", "")) + private val serviceAccountName = + Option(System.getProperty("spark.kubernetes.test.serviceAccountName")) + .getOrElse("default") + val kubernetesClient = defaultClient.inNamespace(namespace) + val clientConfig = kubernetesClient.getConfiguration + + def createNamespace(): Unit = { + defaultClient.namespaces.createNew() + .withNewMetadata() + .withName(namespace) + .endMetadata() + .done() + } + + def deleteNamespace(): Unit = { + defaultClient.namespaces.withName(namespace).delete() + Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) { + val namespaceList = defaultClient + .namespaces() + .list() + .getItems + .asScala + require(!namespaceList.exists(_.getMetadata.getName == namespace)) + } + } + + def newSparkAppConf(): SparkAppConf = { + new SparkAppConf() + .set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}") + .set("spark.kubernetes.namespace", namespace) + .set("spark.executor.memory", "500m") + .set("spark.executor.cores", "1") + .set("spark.executors.instances", "1") + .set("spark.app.name", "spark-test-app") + .set("spark.ui.enabled", "true") + .set("spark.testing", "false") + .set("spark.kubernetes.submission.waitAppCompletion", "false") + .set("spark.kubernetes.authenticate.driver.serviceAccountName", serviceAccountName) + } +} + +private[spark] class SparkAppConf { + + private val map = mutable.Map[String, String]() + + def set(key: String, value: String): SparkAppConf = { + map.put(key, value) + this + } + + def get(key: String): String = map.getOrElse(key, "") + + def setJars(jars: Seq[String]): Unit = set("spark.jars", jars.mkString(",")) + + override def toString: String = map.toString + + def toStringArray: Iterable[String] = map.toList.flatMap(t => List("--conf", s"${t._1}=${t._2}")) +} + +private[spark] case class SparkAppArguments( + mainAppResource: String, + mainClass: String, + appArgs: Array[String]) + +private[spark] object SparkAppLauncher extends Logging { + + def launch( + appArguments: SparkAppArguments, + appConf: SparkAppConf, + timeoutSecs: Int, + sparkHomeDir: Path): Unit = { + val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) + logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") + val commandLine = Array(sparkSubmitExecutable.toFile.getAbsolutePath, + "--deploy-mode", "cluster", + "--class", appArguments.mainClass, + "--master", appConf.get("spark.master") + ) ++ appConf.toStringArray :+ + appArguments.mainAppResource :+ + appArguments.appArgs.mkString(" ") + logInfo(s"Launching a spark app with command line: ${commandLine.mkString(" ")}") + ProcessUtils.executeProcess(commandLine, timeoutSecs) + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala new file mode 100644 index 0000000000000..459c0a4138b86 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import org.apache.log4j.{Logger, LogManager, Priority} + +trait Logging { + + private val log: Logger = LogManager.getLogger(this.getClass) + + protected def logDebug(msg: => String) = if (log.isDebugEnabled) log.debug(msg) + + protected def logInfo(msg: => String) = if (log.isInfoEnabled) log.info(msg) + + protected def logWarning(msg: => String) = if (log.isEnabledFor(Priority.WARN)) log.warn(msg) + + protected def logWarning(msg: => String, throwable: Throwable) = + if (log.isEnabledFor(Priority.WARN)) log.warn(msg, throwable) + + protected def logError(msg: => String) = if (log.isEnabledFor(Priority.ERROR)) log.error(msg) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala new file mode 100644 index 0000000000000..aa6425ddd0353 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.util.concurrent.TimeUnit + +import scala.collection.mutable.ArrayBuffer +import scala.io.Source + +object ProcessUtils extends Logging { + /** + * executeProcess is used to run a command and return the output if it + * completes within timeout seconds. + */ + def executeProcess(fullCommand: Array[String], timeout: Long): Seq[String] = { + val pb = new ProcessBuilder().command(fullCommand: _*) + pb.redirectErrorStream(true) + val proc = pb.start() + val outputLines = new ArrayBuffer[String] + Utils.tryWithResource(proc.getInputStream)( + Source.fromInputStream(_, "UTF-8").getLines().foreach { line => + logInfo(line) + outputLines += line + }) + assert(proc.waitFor(timeout, TimeUnit.SECONDS), + s"Timed out while executing ${fullCommand.mkString(" ")}") + assert(proc.exitValue == 0, s"Failed to execute ${fullCommand.mkString(" ")}") + outputLines + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala new file mode 100644 index 0000000000000..f1fd6dc19ce54 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.util.concurrent.TimeUnit + +import com.google.common.util.concurrent.SettableFuture +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher} +import io.fabric8.kubernetes.client.Watcher.Action +import io.fabric8.kubernetes.client.internal.readiness.Readiness + +private[spark] class SparkReadinessWatcher[T <: HasMetadata] extends Watcher[T] { + + private val signal = SettableFuture.create[Boolean] + + override def eventReceived(action: Action, resource: T): Unit = { + if ((action == Action.MODIFIED || action == Action.ADDED) && + Readiness.isReady(resource)) { + signal.set(true) + } + } + + override def onClose(cause: KubernetesClientException): Unit = {} + + def waitUntilReady(): Boolean = signal.get(60, TimeUnit.SECONDS) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala new file mode 100644 index 0000000000000..c300ca46083d4 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.Closeable +import java.net.URI + +object Utils extends Logging { + + def tryWithResource[R <: Closeable, T](createResource: => R)(f: R => T): T = { + val resource = createResource + try f.apply(resource) finally resource.close() + } + + def tryWithSafeFinally[T](block: => T)(finallyBlock: => Unit): T = { + var originalThrowable: Throwable = null + try { + block + } catch { + case t: Throwable => + // Purposefully not using NonFatal, because even fatal exceptions + // we don't want to have our finallyBlock suppress + originalThrowable = t + throw originalThrowable + } finally { + try { + finallyBlock + } catch { + case t: Throwable => + if (originalThrowable != null) { + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: " + t.getMessage, t) + throw originalThrowable + } else { + throw t + } + } + } + } + + def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { + require(rawMasterURL.startsWith("k8s://"), + "Kubernetes master URL must start with k8s://.") + val masterWithoutK8sPrefix = rawMasterURL.substring("k8s://".length) + + // To handle master URLs, e.g., k8s://host:port. + if (!masterWithoutK8sPrefix.contains("://")) { + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + return s"k8s://$resolvedURL" + } + + val masterScheme = new URI(masterWithoutK8sPrefix).getScheme + val resolvedURL = masterScheme.toLowerCase match { + case "https" => + masterWithoutK8sPrefix + case "http" => + logWarning("Kubernetes master URL uses HTTP instead of HTTPS.") + masterWithoutK8sPrefix + case null => + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + resolvedURL + case _ => + throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) + } + + s"k8s://$resolvedURL" + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala new file mode 100644 index 0000000000000..00b927ff7c327 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.k8s.integrationtest.backend + +import io.fabric8.kubernetes.client.DefaultKubernetesClient + +import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.CloudTestBackend +import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend + +private[spark] trait IntegrationTestBackend { + def initialize(): Unit + def getKubernetesClient: DefaultKubernetesClient + def cleanUp(): Unit = {} +} + +private[spark] object IntegrationTestBackendFactory { + def getTestBackend: IntegrationTestBackend = { + val deployMode = Option(System.getProperty("spark.kubernetes.test.deployMode")) + .getOrElse("minikube") + if (deployMode == "minikube") { + MinikubeTestBackend + } else { + CloudTestBackend + } + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala new file mode 100644 index 0000000000000..2f2aeada12a5b --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.cloud + +import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} + +import org.apache.spark.deploy.k8s.integrationtest.Utils +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object CloudTestBackend extends IntegrationTestBackend { + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + val masterUrl = Option(System.getProperty("spark.kubernetes.test.master")) + .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) + val k8sConf = new ConfigBuilder() + .withApiVersion("v1") + .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) + .build() + defaultClient = new DefaultKubernetesClient(k8sConf) + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala new file mode 100644 index 0000000000000..7145d85fb5a1a --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.minikube + +import java.io.File +import java.nio.file.Paths + +import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} + +import org.apache.spark.deploy.k8s.integrationtest.{Logging, ProcessUtils} + +// TODO support windows +private[spark] object Minikube extends Logging { + + private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60 + + def getMinikubeIp: String = { + val outputs = executeMinikube("ip") + .filter(_.matches("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$")) + assert(outputs.size == 1, "Unexpected amount of output from minikube ip") + outputs.head + } + + def getMinikubeStatus: MinikubeStatus.Value = { + val statusString = executeMinikube("status") + .filter(line => line.contains("minikubeVM: ") || line.contains("minikube:")) + .head + .replaceFirst("minikubeVM: ", "") + .replaceFirst("minikube: ", "") + MinikubeStatus.unapply(statusString) + .getOrElse(throw new IllegalStateException(s"Unknown status $statusString")) + } + + def getKubernetesClient: DefaultKubernetesClient = { + val kubernetesMaster = s"https://${getMinikubeIp}:8443" + val userHome = System.getProperty("user.home") + val kubernetesConf = new ConfigBuilder() + .withApiVersion("v1") + .withMasterUrl(kubernetesMaster) + .withCaCertFile(Paths.get(userHome, ".minikube", "ca.crt").toFile.getAbsolutePath) + .withClientCertFile(Paths.get(userHome, ".minikube", "apiserver.crt").toFile.getAbsolutePath) + .withClientKeyFile(Paths.get(userHome, ".minikube", "apiserver.key").toFile.getAbsolutePath) + .build() + new DefaultKubernetesClient(kubernetesConf) + } + + private def executeMinikube(action: String, args: String*): Seq[String] = { + ProcessUtils.executeProcess( + Array("bash", "-c", s"minikube $action") ++ args, MINIKUBE_STARTUP_TIMEOUT_SECONDS) + } +} + +private[spark] object MinikubeStatus extends Enumeration { + + // The following states are listed according to + // https://github.com/docker/machine/blob/master/libmachine/state/state.go. + val STARTING = status("Starting") + val RUNNING = status("Running") + val PAUSED = status("Paused") + val STOPPING = status("Stopping") + val STOPPED = status("Stopped") + val ERROR = status("Error") + val TIMEOUT = status("Timeout") + val SAVED = status("Saved") + val NONE = status("") + + def status(value: String): Value = new Val(nextId, value) + def unapply(s: String): Option[Value] = values.find(s == _.toString) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala new file mode 100644 index 0000000000000..cb9324179d70e --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.minikube + +import io.fabric8.kubernetes.client.DefaultKubernetesClient + +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object MinikubeTestBackend extends IntegrationTestBackend { + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + val minikubeStatus = Minikube.getMinikubeStatus + require(minikubeStatus == MinikubeStatus.RUNNING, + s"Minikube must be running to use the Minikube backend for integration tests." + + s" Current status is: $minikubeStatus.") + defaultClient = Minikube.getKubernetesClient + } + + override def cleanUp(): Unit = { + super.cleanUp() + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala new file mode 100644 index 0000000000000..a81ef455c6766 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.File + +import com.google.common.base.Charsets +import com.google.common.io.Files + +package object config { + def getTestImageTag: String = { + val imageTagFileProp = System.getProperty("spark.kubernetes.test.imageTagFile") + require(imageTagFileProp != null, "Image tag file must be provided in system properties.") + val imageTagFile = new File(imageTagFileProp) + require(imageTagFile.isFile, s"No file found for image tag at ${imageTagFile.getAbsolutePath}.") + Files.toString(imageTagFile, Charsets.UTF_8).trim + } + + def getTestImageRepo: String = { + val imageRepo = System.getProperty("spark.kubernetes.test.imageRepo") + require(imageRepo != null, "Image repo must be provided in system properties.") + imageRepo + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala new file mode 100644 index 0000000000000..0807a68cd823c --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +package object constants { + val MINIKUBE_TEST_BACKEND = "minikube" + val GCE_TEST_BACKEND = "gce" +} From 20c7b01e4b4dc7a4ece35a9b24e49493c852bc5f Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 09:34:56 -0800 Subject: [PATCH 02/39] Make k8s integration tests build when top-level kubernetes profile selected --- pom.xml | 1 + resource-managers/kubernetes/integration-tests/pom.xml | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b8396166f6b1b..025d1cc386b36 100644 --- a/pom.xml +++ b/pom.xml @@ -2686,6 +2686,7 @@ kubernetes resource-managers/kubernetes/core + resource-managers/kubernetes/integration-tests diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index e3e48fcc07f7d..f1bfada70b473 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -17,6 +17,12 @@ --> 4.0.0 + + org.apache.spark + spark-parent_2.11 + 2.4.0-SNAPSHOT + ../../../pom.xml + spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests @@ -97,7 +103,6 @@ org.scalatest scalatest_${scala.binary.version} - ${scalatest.version} test From 9d238824c8df31d1b8ee86059a693ff7684c4e6c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 10:15:07 -0800 Subject: [PATCH 03/39] Remove LICENSE and copy of mvn wrapper script. Rewrite path for calling mvn wrapper script. --- .../kubernetes/integration-tests/LICENSE | 201 ------------------ .../kubernetes/integration-tests/build/mvn | 29 --- .../dev/dev-run-integration-tests.sh | 2 +- 3 files changed, 1 insertion(+), 231 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/LICENSE delete mode 100755 resource-managers/kubernetes/integration-tests/build/mvn diff --git a/resource-managers/kubernetes/integration-tests/LICENSE b/resource-managers/kubernetes/integration-tests/LICENSE deleted file mode 100644 index 261eeb9e9f8b2..0000000000000 --- a/resource-managers/kubernetes/integration-tests/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/resource-managers/kubernetes/integration-tests/build/mvn b/resource-managers/kubernetes/integration-tests/build/mvn deleted file mode 100755 index 87e5b58640c5c..0000000000000 --- a/resource-managers/kubernetes/integration-tests/build/mvn +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BUILD_DIR=$(dirname $0) - -MVN_RUNNER=$BUILD_DIR/run-mvn - -if [ ! -f $MVN_RUNNER ]; -then - curl -s --progress-bar https://raw.githubusercontent.com/apache/spark/master/build/mvn > $MVN_RUNNER - chmod +x $MVN_RUNNER -fi -source $MVN_RUNNER diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 33879d5fc8b2a..eb9236981cfcf 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -107,4 +107,4 @@ then properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER ) fi -build/mvn integration-test ${properties[@]} +../../../build/mvn integration-test ${properties[@]} From f42ca7f535c1ebdd34480097c13fe097a9260edd Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 11:13:00 -0800 Subject: [PATCH 04/39] Fix scala style issues --- .../k8s/integrationtest/KubernetesSuite.scala | 215 +++++++++--------- .../KubernetesTestComponents.scala | 2 +- 2 files changed, 109 insertions(+), 108 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 89c8a91c4ab58..6203e29785820 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -22,16 +22,17 @@ import java.util.UUID import java.util.regex.Pattern import scala.collection.JavaConverters._ + import com.google.common.io.PatternFilenameFilter import io.fabric8.kubernetes.api.model.{Container, Pod} -import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite} +import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.{Minutes, Seconds, Span} import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} import org.apache.spark.deploy.k8s.integrationtest.config._ -private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll with BeforeAndAfter { +private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter { import KubernetesSuite._ @@ -101,111 +102,111 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit runSparkPiAndVerifyCompletion() } - test("Run SparkPi with a very long application name.") { - sparkAppConf.set("spark.app.name", "long" * 40) - runSparkPiAndVerifyCompletion() - } - - test("Run SparkPi with a master URL without a scheme.") { - val url = kubernetesTestComponents.kubernetesClient.getMasterUrl - val k8sMasterUrl = if (url.getPort < 0) { - s"k8s://${url.getHost}" - } else { - s"k8s://${url.getHost}:${url.getPort}" - } - sparkAppConf.set("spark.master", k8sMasterUrl) - runSparkPiAndVerifyCompletion() - } - - test("Run SparkPi with an argument.") { - runSparkPiAndVerifyCompletion(appArgs = Array("5")) - } - - test("Run SparkPi with custom labels, annotations, and environment variables.") { - sparkAppConf - .set("spark.kubernetes.driver.label.label1", "label1-value") - .set("spark.kubernetes.driver.label.label2", "label2-value") - .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") - .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") - .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") - .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") - .set("spark.kubernetes.executor.label.label1", "label1-value") - .set("spark.kubernetes.executor.label.label2", "label2-value") - .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") - .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") - .set("spark.executorEnv.ENV1", "VALUE1") - .set("spark.executorEnv.ENV2", "VALUE2") - - runSparkPiAndVerifyCompletion( - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkCustomSettings(driverPod) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkCustomSettings(executorPod) - }) - } - - test("Run SparkPi with a test secret mounted into the driver and executor pods") { - val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") - createTestSecret(secretName) - - sparkAppConf - .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - - try { - runSparkPiAndVerifyCompletion( - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkTestSecret(secretName, driverPod) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkTestSecret(secretName, executorPod) - }) - } finally { - deleteTestSecret(secretName) - } - } - - test("Run PageRank using remote data file") { - sparkAppConf - .set("spark.kubernetes.mountDependencies.filesDownloadDir", - CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) - .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - runSparkPageRankAndVerifyCompletion( - appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) - } - - test("Run PageRank using remote data file with test secret mounted into the driver and " + - "executors") { - val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") - createTestSecret(secretName) - - sparkAppConf - .set("spark.kubernetes.mountDependencies.filesDownloadDir", - CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) - .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - - try { - runSparkPageRankAndVerifyCompletion( - appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkTestSecret(secretName, driverPod, withInitContainer = true) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkTestSecret(secretName, executorPod, withInitContainer = true) - }) - } finally { - deleteTestSecret(secretName) - } - } +// test("Run SparkPi with a very long application name.") { +// sparkAppConf.set("spark.app.name", "long" * 40) +// runSparkPiAndVerifyCompletion() +// } +// +// test("Run SparkPi with a master URL without a scheme.") { +// val url = kubernetesTestComponents.kubernetesClient.getMasterUrl +// val k8sMasterUrl = if (url.getPort < 0) { +// s"k8s://${url.getHost}" +// } else { +// s"k8s://${url.getHost}:${url.getPort}" +// } +// sparkAppConf.set("spark.master", k8sMasterUrl) +// runSparkPiAndVerifyCompletion() +// } +// +// test("Run SparkPi with an argument.") { +// runSparkPiAndVerifyCompletion(appArgs = Array("5")) +// } +// +// test("Run SparkPi with custom labels, annotations, and environment variables.") { +// sparkAppConf +// .set("spark.kubernetes.driver.label.label1", "label1-value") +// .set("spark.kubernetes.driver.label.label2", "label2-value") +// .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") +// .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") +// .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") +// .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") +// .set("spark.kubernetes.executor.label.label1", "label1-value") +// .set("spark.kubernetes.executor.label.label2", "label2-value") +// .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") +// .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") +// .set("spark.executorEnv.ENV1", "VALUE1") +// .set("spark.executorEnv.ENV2", "VALUE2") +// +// runSparkPiAndVerifyCompletion( +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkCustomSettings(driverPod) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkCustomSettings(executorPod) +// }) +// } +// +// test("Run SparkPi with a test secret mounted into the driver and executor pods") { +// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") +// createTestSecret(secretName) +// +// sparkAppConf +// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// +// try { +// runSparkPiAndVerifyCompletion( +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkTestSecret(secretName, driverPod) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkTestSecret(secretName, executorPod) +// }) +// } finally { +// deleteTestSecret(secretName) +// } +// } +// +// test("Run PageRank using remote data file") { +// sparkAppConf +// .set("spark.kubernetes.mountDependencies.filesDownloadDir", +// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) +// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) +// runSparkPageRankAndVerifyCompletion( +// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) +// } +// +// test("Run PageRank using remote data file with test secret mounted into the driver and " + +// "executors") { +// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") +// createTestSecret(secretName) +// +// sparkAppConf +// .set("spark.kubernetes.mountDependencies.filesDownloadDir", +// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) +// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) +// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// +// try { +// runSparkPageRankAndVerifyCompletion( +// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkTestSecret(secretName, driverPod, withInitContainer = true) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkTestSecret(secretName, executorPod, withInitContainer = true) +// }) +// } finally { +// deleteTestSecret(secretName) +// } +// } private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index b9c87d59fe27b..576bf6a2c5d8b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -19,8 +19,8 @@ package org.apache.spark.deploy.k8s.integrationtest import java.nio.file.{Path, Paths} import java.util.UUID -import scala.collection.mutable import scala.collection.JavaConverters._ +import scala.collection.mutable import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually From 58952ac5064763ced45db1899349b12bb4fa030c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 13:41:45 -0800 Subject: [PATCH 05/39] Ignore dist/ for style checks --- dev/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tox.ini b/dev/tox.ini index 583c1eaaa966b..28dad8f3b5c7c 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -16,4 +16,4 @@ [pycodestyle] ignore=E402,E731,E241,W503,E226,E722,E741,E305 max-line-length=100 -exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/* +exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/* From 84a7779f50a0e4d4ca3ef01f5c3e430c681e569c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 13:42:55 -0800 Subject: [PATCH 06/39] Fixes for scala style --- .../kubernetes/integration-tests/pom.xml | 15 ++++++++++++++- .../k8s/integrationtest/KubernetesSuite.scala | 4 +++- .../KubernetesTestComponents.scala | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index f1bfada70b473..c3d06dadebe20 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -26,7 +26,6 @@ spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests - 0.1-SNAPSHOT 3.3.9 3.5 @@ -57,6 +56,20 @@ Spark Project Kubernetes Integration Tests + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test-jar + test + + commons-logging commons-logging diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 6203e29785820..7353e82569c57 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -29,10 +29,12 @@ import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.{Minutes, Seconds, Span} +import org.apache.spark.SparkFunSuite import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} import org.apache.spark.deploy.k8s.integrationtest.config._ -private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter { +private[spark] class KubernetesSuite extends SparkFunSuite + with BeforeAndAfterAll with BeforeAndAfter { import KubernetesSuite._ diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 576bf6a2c5d8b..2df59b6a3373e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -21,6 +21,7 @@ import java.util.UUID import scala.collection.JavaConverters._ import scala.collection.mutable + import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually From 7764b07c203c8b80ed63860b961d754490092b36 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:08:37 -0700 Subject: [PATCH 07/39] Remove repository cloning behavior and allow script to be called from other directories --- .../integration-tests/dev/dev-run-integration-tests.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index eb9236981cfcf..2b47a42890c38 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -16,10 +16,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests + +cd "${TEST_ROOT_DIR}" source ./include/util.sh -TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests BRANCH="master" SPARK_REPO="https://github.com/apache/spark" SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" @@ -77,12 +79,6 @@ while (( "$#" )); do shift done -if [[ $SPARK_TGZ == "N/A" ]]; -then - echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." - clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH -fi - cd $TEST_ROOT_DIR properties=( From aa72b6e947b588d834ed4840f24c3b3483f73d9f Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:09:54 -0700 Subject: [PATCH 08/39] Remove config options that were only used during repo clone process --- .../dev/dev-run-integration-tests.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 2b47a42890c38..754588194d607 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -22,9 +22,6 @@ cd "${TEST_ROOT_DIR}" source ./include/util.sh -BRANCH="master" -SPARK_REPO="https://github.com/apache/spark" -SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" DEPLOY_MODE="minikube" IMAGE_REPO="docker.io/kubespark" SPARK_TGZ="N/A" @@ -36,14 +33,6 @@ SERVICE_ACCOUNT= # Parse arguments while (( "$#" )); do case $1 in - --spark-branch) - BRANCH="$2" - shift - ;; - --spark-repo) - SPARK_REPO="$2" - shift - ;; --image-repo) IMAGE_REPO="$2" shift From 81956aac94dc9b76bc4a2af5dfe9a5db6fe788ef Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:52:35 -0700 Subject: [PATCH 09/39] Remove K8s cloud-based backend testing support from this PR --- .../backend/IntegrationTestBackend.scala | 8 ++-- .../backend/cloud/CloudTestBackend.scala | 41 ------------------- 2 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index 00b927ff7c327..e9a202677654d 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -19,7 +19,6 @@ package org.apache.spark.deploy.k8s.integrationtest.backend import io.fabric8.kubernetes.client.DefaultKubernetesClient -import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.CloudTestBackend import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend private[spark] trait IntegrationTestBackend { @@ -29,13 +28,16 @@ private[spark] trait IntegrationTestBackend { } private[spark] object IntegrationTestBackendFactory { + val DeployModeConfigKey = "spark.kubernetes.test.deployMode" + def getTestBackend: IntegrationTestBackend = { - val deployMode = Option(System.getProperty("spark.kubernetes.test.deployMode")) + val deployMode = Option(System.getProperty(DeployModeConfigKey)) .getOrElse("minikube") if (deployMode == "minikube") { MinikubeTestBackend } else { - CloudTestBackend + throw new IllegalArgumentException( + "Invalid " + DeployModeConfigKey + ": " + deployMode) } } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala deleted file mode 100644 index 2f2aeada12a5b..0000000000000 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.deploy.k8s.integrationtest.backend.cloud - -import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} - -import org.apache.spark.deploy.k8s.integrationtest.Utils -import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend - -private[spark] object CloudTestBackend extends IntegrationTestBackend { - - private var defaultClient: DefaultKubernetesClient = _ - - override def initialize(): Unit = { - val masterUrl = Option(System.getProperty("spark.kubernetes.test.master")) - .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) - val k8sConf = new ConfigBuilder() - .withApiVersion("v1") - .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) - .build() - defaultClient = new DefaultKubernetesClient(k8sConf) - } - - override def getKubernetesClient: DefaultKubernetesClient = { - defaultClient - } -} From ba720733fa24ce3cebe6bfe3eafd0d9621df5388 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:12:08 -0700 Subject: [PATCH 10/39] Update README.md excluding cloning and building logic --- .../kubernetes/integration-tests/README.md | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 97cb8ab42e7a0..b3863e6b7d1af 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -23,21 +23,6 @@ You can download Minikube [here](https://github.com/kubernetes/minikube/releases Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. -## Use a non-local cluster - -To use your own cluster running in the cloud, set the following: - -* `--deploy-mode cloud` to indicate that the test is connecting to a remote cluster instead of Minikube, -* `--spark-master ` - set `` to the externally accessible Kubernetes cluster URL, -* `--image-repo ` - set `` to a write-accessible Docker image repository that provides the images for your cluster. The framework assumes your local Docker client can push to this repository. - -Therefore the command looks like this: - - dev/dev-run-integration-tests.sh \ - --deploy-mode cloud \ - --spark-master https://example.com:8443/apiserver \ - --image-repo docker.example.com/spark-images - ## Re-using Docker Images By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, @@ -50,28 +35,13 @@ where if you still want to use images that were built before by the test framewo dev/dev-run-integration-tests.sh --image-tag $(cat target/imageTag.txt) -## Customizing the Spark Source Code to Test - -By default, the test framework will test the master branch of Spark from [here](https://github.com/apache/spark). You -can specify the following options to test against different source versions of Spark: - -* `--spark-repo ` - set `` to the git or http URI of the Spark git repository to clone -* `--spark-branch ` - set `` to the branch of the repository to build. - - -An example: - - dev/dev-run-integration-tests.sh \ - --spark-repo https://github.com/apache-spark-on-k8s/spark \ - --spark-branch new-feature +## Spark Distribution Under Test -Additionally, you can use a pre-built Spark distribution. In this case, the repository is not cloned at all, and no -source code has to be compiled. +The Spark code to test is handed to the integration test system via a tarball. Here is the option that is used to specify the tarball: * `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. -When the tests are cloning a repository and building it, the Spark distribution is placed in `target/spark/spark-.tgz`. -Reuse this tarball to save a significant amount of time if you are iterating on the development of these integration tests. +TODO: Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current tree. ## Customizing the Namespace and Service Account @@ -79,4 +49,4 @@ Reuse this tarball to save a significant amount of time if you are iterating on * `--service-account ` - set `` to the name of the Kubernetes service account to use in the namespace specified by the `--namespace`. The service account is expected to have permissions to get, list, watch, and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account -in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. \ No newline at end of file +in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. From 13721f69a21c91c0b42a5471b50b8f53e4c7808f Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:13:17 -0700 Subject: [PATCH 11/39] Remove unnecessary cloning and building code for the Spark repo --- .../dev/dev-run-integration-tests.sh | 2 - .../integration-tests/include/util.sh | 43 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/include/util.sh diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 754588194d607..ea893fa39eede 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -20,8 +20,6 @@ TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/inte cd "${TEST_ROOT_DIR}" -source ./include/util.sh - DEPLOY_MODE="minikube" IMAGE_REPO="docker.io/kubespark" SPARK_TGZ="N/A" diff --git a/resource-managers/kubernetes/integration-tests/include/util.sh b/resource-managers/kubernetes/integration-tests/include/util.sh deleted file mode 100644 index 49b9114854120..0000000000000 --- a/resource-managers/kubernetes/integration-tests/include/util.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -clone_build_spark() { - spark_repo=$1 - spark_repo_local_dir=$2 - branch=$3 - pushd . - - # clone spark distribution if needed. - # TODO(ssuchter): This code, that does a checkout of a spark repo, - # made more sense when this script was in the repo - # https://github.com/apache-spark-on-k8s/spark-integration - # but now we shouldn't check out another copy of spark, we should just - # build in the copy that is checked out already. - if [ -d "$spark_repo_local_dir" ]; - then - (cd $spark_repo_local_dir && git fetch origin $branch); - else - mkdir -p $spark_repo_local_dir; - git clone -b $branch --single-branch $spark_repo $spark_repo_local_dir; - fi - cd $spark_repo_local_dir - git checkout -B $branch origin/$branch - ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests; - SPARK_TGZ=$(find $spark_repo_local_dir -name spark-*.tgz) - - popd -} From 5ffa464c65d9caaae86df059a75f99a893acdcf9 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:17:46 -0700 Subject: [PATCH 12/39] Remove e2e-prow.sh, which isn't appropriate for this PR --- .../integration-tests/e2e/e2e-prow.sh | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100755 resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh diff --git a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh deleted file mode 100755 index 501f350943e73..0000000000000 --- a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -### This script is used by Kubernetes Test Infrastructure to run integration tests. -### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow - -set -ex - -# set cwd correctly -cd "$(dirname "$0")/../" - -# Include requisite scripts -source ./include/util.sh - -TEST_ROOT_DIR=$(git rev-parse --show-toplevel) -BRANCH="master" -SPARK_REPO="https://github.com/apache/spark" -SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" - -## Install basic dependencies -## These are for the kubekins-e2e environment in https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e -echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -apt-get update && apt-get install -y curl wget git tar uuid-runtime -apt-get install -t jessie-backports -y openjdk-8-jdk - -# Set up config. -MASTER=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") - -# Special GCP project for publishing docker images built by test. -IMAGE_REPO="gcr.io/spark-testing-191023" - -# Cloning the spark distribution -echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." -clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH - -# Spark distribution -properties=( - -Dspark.kubernetes.test.master=k8s://$MASTER \ - -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ - -Dspark.kubernetes.test.sparkTgz="$SPARK_TGZ" \ - -Dspark.kubernetes.test.deployMode=cloud \ - -Dspark.kubernetes.test.namespace=spark \ - -Dspark.kubernetes.test.serviceAccountName=spark-sa -) - -# Run kubectl commands and create appropriate roles -kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com -kubectl create -f ./dev/spark-rbac.yaml - -# Run tests. -echo "Starting test with ${properties[@]}" -build/mvn integration-test "${properties[@]}" || : - -# Copy out the junit xml files for consumption by k8s test-infra. -ls -1 ./target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done From dbce27578439528ab699945f724c0c4c2888c985 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:49:11 -0700 Subject: [PATCH 13/39] Remove unused code relating to Kerberos, which doesn't belong in this PR --- .../k8s/integrationtest/KubernetesSuite.scala | 163 ------------------ 1 file changed, 163 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 7353e82569c57..8505f70e13e2b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -104,112 +104,6 @@ private[spark] class KubernetesSuite extends SparkFunSuite runSparkPiAndVerifyCompletion() } -// test("Run SparkPi with a very long application name.") { -// sparkAppConf.set("spark.app.name", "long" * 40) -// runSparkPiAndVerifyCompletion() -// } -// -// test("Run SparkPi with a master URL without a scheme.") { -// val url = kubernetesTestComponents.kubernetesClient.getMasterUrl -// val k8sMasterUrl = if (url.getPort < 0) { -// s"k8s://${url.getHost}" -// } else { -// s"k8s://${url.getHost}:${url.getPort}" -// } -// sparkAppConf.set("spark.master", k8sMasterUrl) -// runSparkPiAndVerifyCompletion() -// } -// -// test("Run SparkPi with an argument.") { -// runSparkPiAndVerifyCompletion(appArgs = Array("5")) -// } -// -// test("Run SparkPi with custom labels, annotations, and environment variables.") { -// sparkAppConf -// .set("spark.kubernetes.driver.label.label1", "label1-value") -// .set("spark.kubernetes.driver.label.label2", "label2-value") -// .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") -// .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") -// .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") -// .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") -// .set("spark.kubernetes.executor.label.label1", "label1-value") -// .set("spark.kubernetes.executor.label.label2", "label2-value") -// .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") -// .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") -// .set("spark.executorEnv.ENV1", "VALUE1") -// .set("spark.executorEnv.ENV2", "VALUE2") -// -// runSparkPiAndVerifyCompletion( -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkCustomSettings(driverPod) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkCustomSettings(executorPod) -// }) -// } -// -// test("Run SparkPi with a test secret mounted into the driver and executor pods") { -// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") -// createTestSecret(secretName) -// -// sparkAppConf -// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// -// try { -// runSparkPiAndVerifyCompletion( -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkTestSecret(secretName, driverPod) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkTestSecret(secretName, executorPod) -// }) -// } finally { -// deleteTestSecret(secretName) -// } -// } -// -// test("Run PageRank using remote data file") { -// sparkAppConf -// .set("spark.kubernetes.mountDependencies.filesDownloadDir", -// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) -// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) -// runSparkPageRankAndVerifyCompletion( -// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) -// } -// -// test("Run PageRank using remote data file with test secret mounted into the driver and " + -// "executors") { -// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") -// createTestSecret(secretName) -// -// sparkAppConf -// .set("spark.kubernetes.mountDependencies.filesDownloadDir", -// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) -// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) -// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// -// try { -// runSparkPageRankAndVerifyCompletion( -// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkTestSecret(secretName, driverPod, withInitContainer = true) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkTestSecret(secretName, executorPod, withInitContainer = true) -// }) -// } finally { -// deleteTestSecret(secretName) -// } -// } - private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, driverPodChecker: Pod => Unit = doBasicDriverPodCheck, @@ -324,51 +218,6 @@ private[spark] class KubernetesSuite extends SparkFunSuite .get() == null) } } - - private def createTestSecret(secretName: String): Unit = { - kubernetesTestComponents.kubernetesClient.secrets - .createNew() - .editOrNewMetadata() - .withName(secretName) - .endMetadata() - .addToStringData(TEST_SECRET_KEY, TEST_SECRET_VALUE) - .done() - } - - private def checkTestSecret( - secretName: String, - pod: Pod, - withInitContainer: Boolean = false): Unit = { - val testSecretVolume = pod.getSpec.getVolumes.asScala.filter { volume => - volume.getName == s"$secretName-volume" - } - assert(testSecretVolume.size === 1) - assert(testSecretVolume.head.getSecret.getSecretName === secretName) - - checkTestSecretInContainer(secretName, pod.getSpec.getContainers.get(0)) - - if (withInitContainer) { - checkTestSecretInContainer(secretName, pod.getSpec.getInitContainers.get(0)) - } - } - - private def checkTestSecretInContainer(secretName: String, container: Container): Unit = { - val testSecret = container.getVolumeMounts.asScala.filter { mount => - mount.getName == s"$secretName-volume" - } - assert(testSecret.size === 1) - assert(testSecret.head.getMountPath === TEST_SECRET_MOUNT_PATH) - } - - private def deleteTestSecret(secretName: String): Unit = { - kubernetesTestComponents.kubernetesClient.secrets - .withName(secretName) - .delete() - - Eventually.eventually(TIMEOUT, INTERVAL) { - assert(kubernetesTestComponents.kubernetesClient.secrets.withName(secretName).get() == null) - } - } } private[spark] object KubernetesSuite { @@ -378,17 +227,5 @@ private[spark] object KubernetesSuite { val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" - val TEST_SECRET_NAME_PREFIX = "test-secret-" - val TEST_SECRET_KEY = "test-key" - val TEST_SECRET_VALUE = "test-data" - val TEST_SECRET_MOUNT_PATH = "/etc/secrets" - - val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" - - val REMOTE_PAGE_RANK_DATA_FILE = - "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" - val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = - s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" - case object ShuffleNotReadyException extends Exception } From 365d6bc65dde0e47f29dcaa0bc18c633730eb07d Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 28 Feb 2018 12:10:46 -0800 Subject: [PATCH 14/39] Initial checkin of k8s integration tests. These tests were developed in the https://github.com/apache-spark-on-k8s/spark-integration repo by several contributors. This is a copy of the current state into the main apache spark repo. The only changes from the current spark-integration repo state are: * Move the files from the repo root into resource-managers/kubernetes/integration-tests * Add a reference to these tests in the root README.md * Fix a path reference in dev/dev-run-integration-tests.sh * Add a TODO in include/util.sh --- README.md | 2 + .../kubernetes/integration-tests/LICENSE | 201 +++++++++ .../kubernetes/integration-tests/README.md | 82 ++++ .../kubernetes/integration-tests/build/mvn | 29 ++ .../dev/dev-run-integration-tests.sh | 110 +++++ .../integration-tests/dev/spark-rbac.yaml | 52 +++ .../integration-tests/e2e/e2e-prow.sh | 69 ++++ .../integration-tests/include/util.sh | 43 ++ .../kubernetes/integration-tests/pom.xml | 212 ++++++++++ .../scripts/setup-integration-test-env.sh | 91 ++++ .../src/test/resources/log4j.properties | 31 ++ .../k8s/integrationtest/KubernetesSuite.scala | 391 ++++++++++++++++++ .../KubernetesTestComponents.scala | 115 ++++++ .../deploy/k8s/integrationtest/Logging.scala | 35 ++ .../k8s/integrationtest/ProcessUtils.scala | 44 ++ .../SparkReadinessWatcher.scala | 41 ++ .../deploy/k8s/integrationtest/Utils.scala | 86 ++++ .../backend/IntegrationTestBackend.scala | 41 ++ .../backend/cloud/CloudTestBackend.scala | 41 ++ .../backend/minikube/Minikube.scala | 83 ++++ .../minikube/MinikubeTestBackend.scala | 42 ++ .../deploy/k8s/integrationtest/config.scala | 38 ++ .../k8s/integrationtest/constants.scala | 22 + 23 files changed, 1901 insertions(+) create mode 100644 resource-managers/kubernetes/integration-tests/LICENSE create mode 100644 resource-managers/kubernetes/integration-tests/README.md create mode 100755 resource-managers/kubernetes/integration-tests/build/mvn create mode 100755 resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh create mode 100644 resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml create mode 100755 resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh create mode 100644 resource-managers/kubernetes/integration-tests/include/util.sh create mode 100644 resource-managers/kubernetes/integration-tests/pom.xml create mode 100755 resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh create mode 100644 resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala create mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala diff --git a/README.md b/README.md index 1e521a7e7b178..531d330234062 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ can be run using: Please see the guidance on how to [run tests for a module, or individual tests](http://spark.apache.org/developer-tools.html#individual-tests). +There is also a Kubernetes integration test, see resource-managers/kubernetes/integration-tests/README.md + ## A Note About Hadoop Versions Spark uses the Hadoop core library to talk to HDFS and other Hadoop-supported diff --git a/resource-managers/kubernetes/integration-tests/LICENSE b/resource-managers/kubernetes/integration-tests/LICENSE new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md new file mode 100644 index 0000000000000..97cb8ab42e7a0 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -0,0 +1,82 @@ +--- +layout: global +title: Spark on Kubernetes Integration Tests +--- + +# Running the Kubernetes Integration Tests + +Note that the integration test framework is currently being heavily revised and +is subject to change. Note that currently the integration tests only run with Java 8. + +The simplest way to run the integration tests is to install and run Minikube, then run the following: + + dev/dev-run-integration-tests.sh + +The minimum tested version of Minikube is 0.23.0. The kube-dns addon must be enabled. Minikube should +run with a minimum of 3 CPUs and 4G of memory: + + minikube start --cpus 3 --memory 4096 + +You can download Minikube [here](https://github.com/kubernetes/minikube/releases). + +# Integration test customization + +Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. + +## Use a non-local cluster + +To use your own cluster running in the cloud, set the following: + +* `--deploy-mode cloud` to indicate that the test is connecting to a remote cluster instead of Minikube, +* `--spark-master ` - set `` to the externally accessible Kubernetes cluster URL, +* `--image-repo ` - set `` to a write-accessible Docker image repository that provides the images for your cluster. The framework assumes your local Docker client can push to this repository. + +Therefore the command looks like this: + + dev/dev-run-integration-tests.sh \ + --deploy-mode cloud \ + --spark-master https://example.com:8443/apiserver \ + --image-repo docker.example.com/spark-images + +## Re-using Docker Images + +By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, +and it is written to file at `target/imageTag.txt`. To reuse the images built in a previous run, or to use a Docker image tag +that you have built by other means already, pass the tag to the test script: + + dev/dev-run-integration-tests.sh --image-tag + +where if you still want to use images that were built before by the test framework: + + dev/dev-run-integration-tests.sh --image-tag $(cat target/imageTag.txt) + +## Customizing the Spark Source Code to Test + +By default, the test framework will test the master branch of Spark from [here](https://github.com/apache/spark). You +can specify the following options to test against different source versions of Spark: + +* `--spark-repo ` - set `` to the git or http URI of the Spark git repository to clone +* `--spark-branch ` - set `` to the branch of the repository to build. + + +An example: + + dev/dev-run-integration-tests.sh \ + --spark-repo https://github.com/apache-spark-on-k8s/spark \ + --spark-branch new-feature + +Additionally, you can use a pre-built Spark distribution. In this case, the repository is not cloned at all, and no +source code has to be compiled. + +* `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. + +When the tests are cloning a repository and building it, the Spark distribution is placed in `target/spark/spark-.tgz`. +Reuse this tarball to save a significant amount of time if you are iterating on the development of these integration tests. + +## Customizing the Namespace and Service Account + +* `--namespace ` - set `` to the namespace in which the tests should be run. +* `--service-account ` - set `` to the name of the Kubernetes service account to +use in the namespace specified by the `--namespace`. The service account is expected to have permissions to get, list, watch, +and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account +in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/build/mvn b/resource-managers/kubernetes/integration-tests/build/mvn new file mode 100755 index 0000000000000..87e5b58640c5c --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/build/mvn @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BUILD_DIR=$(dirname $0) + +MVN_RUNNER=$BUILD_DIR/run-mvn + +if [ ! -f $MVN_RUNNER ]; +then + curl -s --progress-bar https://raw.githubusercontent.com/apache/spark/master/build/mvn > $MVN_RUNNER + chmod +x $MVN_RUNNER +fi +source $MVN_RUNNER diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh new file mode 100755 index 0000000000000..33879d5fc8b2a --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +source ./include/util.sh + +TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests +BRANCH="master" +SPARK_REPO="https://github.com/apache/spark" +SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +SPARK_TGZ="N/A" +IMAGE_TAG="N/A" +SPARK_MASTER= +NAMESPACE= +SERVICE_ACCOUNT= + +# Parse arguments +while (( "$#" )); do + case $1 in + --spark-branch) + BRANCH="$2" + shift + ;; + --spark-repo) + SPARK_REPO="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + --spark-master) + SPARK_MASTER="$2" + shift + ;; + --namespace) + NAMESPACE="$2" + shift + ;; + --service-account) + SERVICE_ACCOUNT="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." + clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH +fi + +cd $TEST_ROOT_DIR + +properties=( + -Dspark.kubernetes.test.sparkTgz=$SPARK_TGZ \ + -Dspark.kubernetes.test.imageTag=$IMAGE_TAG \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.deployMode=$DEPLOY_MODE +) + +if [ -n $NAMESPACE ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.namespace=$NAMESPACE ) +fi + +if [ -n $SERVICE_ACCOUNT ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.serviceAccountName=$SERVICE_ACCOUNT ) +fi + +if [ -n $SPARK_MASTER ]; +then + properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER ) +fi + +build/mvn integration-test ${properties[@]} diff --git a/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml b/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml new file mode 100644 index 0000000000000..a4c242f2f2645 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/dev/spark-rbac.yaml @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: Namespace +metadata: + name: spark +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: spark-sa + namespace: spark +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: spark-role +rules: +- apiGroups: + - "" + resources: + - "pods" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: spark-role-binding +subjects: +- kind: ServiceAccount + name: spark-sa + namespace: spark +roleRef: + kind: ClusterRole + name: spark-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh new file mode 100755 index 0000000000000..501f350943e73 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### This script is used by Kubernetes Test Infrastructure to run integration tests. +### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow + +set -ex + +# set cwd correctly +cd "$(dirname "$0")/../" + +# Include requisite scripts +source ./include/util.sh + +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +BRANCH="master" +SPARK_REPO="https://github.com/apache/spark" +SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" + +## Install basic dependencies +## These are for the kubekins-e2e environment in https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e +echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +apt-get update && apt-get install -y curl wget git tar uuid-runtime +apt-get install -t jessie-backports -y openjdk-8-jdk + +# Set up config. +MASTER=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") + +# Special GCP project for publishing docker images built by test. +IMAGE_REPO="gcr.io/spark-testing-191023" + +# Cloning the spark distribution +echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." +clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH + +# Spark distribution +properties=( + -Dspark.kubernetes.test.master=k8s://$MASTER \ + -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ + -Dspark.kubernetes.test.sparkTgz="$SPARK_TGZ" \ + -Dspark.kubernetes.test.deployMode=cloud \ + -Dspark.kubernetes.test.namespace=spark \ + -Dspark.kubernetes.test.serviceAccountName=spark-sa +) + +# Run kubectl commands and create appropriate roles +kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com +kubectl create -f ./dev/spark-rbac.yaml + +# Run tests. +echo "Starting test with ${properties[@]}" +build/mvn integration-test "${properties[@]}" || : + +# Copy out the junit xml files for consumption by k8s test-infra. +ls -1 ./target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done diff --git a/resource-managers/kubernetes/integration-tests/include/util.sh b/resource-managers/kubernetes/integration-tests/include/util.sh new file mode 100644 index 0000000000000..49b9114854120 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/include/util.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +clone_build_spark() { + spark_repo=$1 + spark_repo_local_dir=$2 + branch=$3 + pushd . + + # clone spark distribution if needed. + # TODO(ssuchter): This code, that does a checkout of a spark repo, + # made more sense when this script was in the repo + # https://github.com/apache-spark-on-k8s/spark-integration + # but now we shouldn't check out another copy of spark, we should just + # build in the copy that is checked out already. + if [ -d "$spark_repo_local_dir" ]; + then + (cd $spark_repo_local_dir && git fetch origin $branch); + else + mkdir -p $spark_repo_local_dir; + git clone -b $branch --single-branch $spark_repo $spark_repo_local_dir; + fi + cd $spark_repo_local_dir + git checkout -B $branch origin/$branch + ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests; + SPARK_TGZ=$(find $spark_repo_local_dir -name spark-*.tgz) + + popd +} diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml new file mode 100644 index 0000000000000..e3e48fcc07f7d --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -0,0 +1,212 @@ + + + + 4.0.0 + + spark-kubernetes-integration-tests_2.11 + spark-kubernetes-integration-tests + 0.1-SNAPSHOT + + 3.3.9 + 3.5 + 1.1.1 + 5.0.2 + 1.3.0 + 1.4.0 + + 18.0 + 1.3.9 + 3.0.0 + 1.2.17 + 2.11.8 + 2.11 + 3.2.2 + 2.2.6 + 1.0 + 1.7.24 + kubernetes-integration-tests + ${project.build.directory}/spark-dist-unpacked + N/A + ${project.build.directory}/imageTag.txt + minikube + docker.io/kubespark + + + jar + Spark Project Kubernetes Integration Tests + + + + commons-logging + commons-logging + ${commons-logging.version} + + + com.google.code.findbugs + jsr305 + ${jsr305.version} + + + com.google.guava + guava + test + + ${guava.version} + + + com.spotify + docker-client + ${docker-client.version} + test + + + io.fabric8 + kubernetes-client + ${kubernetes-client.version} + + + log4j + log4j + ${log4j.version} + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + org.scala-lang + scala-library + ${scala.version} + + + org.scalatest + scalatest_${scala.binary.version} + ${scalatest.version} + test + + + org.slf4j + slf4j-log4j12 + ${slf4j-log4j12.version} + test + + + + + + + net.alchim31.maven + scala-maven-plugin + ${scala-maven-plugin.version} + + + + compile + testCompile + + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + setup-integration-test-env + pre-integration-test + + exec + + + scripts/setup-integration-test-env.sh + + --unpacked-spark-tgz + ${spark.kubernetes.test.unpackSparkDir} + + --image-repo + ${spark.kubernetes.test.imageRepo} + + --image-tag + ${spark.kubernetes.test.imageTag} + + --image-tag-output-file + ${spark.kubernetes.test.imageTagFile} + + --deploy-mode + ${spark.kubernetes.test.deployMode} + + --spark-tgz + ${spark.kubernetes.test.sparkTgz} + + + + + + + + org.scalatest + scalatest-maven-plugin + ${scalatest-maven-plugin.version} + + ${project.build.directory}/surefire-reports + . + SparkTestSuite.txt + -ea -Xmx3g -XX:ReservedCodeCacheSize=512m ${extraScalaTestArgs} + + + file:src/test/resources/log4j.properties + true + ${spark.kubernetes.test.imageTagFile} + ${spark.kubernetes.test.unpackSparkDir} + ${spark.kubernetes.test.imageRepo} + ${spark.kubernetes.test.deployMode} + ${spark.kubernetes.test.master} + ${spark.kubernetes.test.namespace} + ${spark.kubernetes.test.serviceAccountName} + + ${test.exclude.tags} + + + + test + + test + + + + (?<!Suite) + + + + integration-test + integration-test + + test + + + + + + + + + diff --git a/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh new file mode 100755 index 0000000000000..ccfb8e767c529 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/scripts/setup-integration-test-env.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +TEST_ROOT_DIR=$(git rev-parse --show-toplevel) +UNPACKED_SPARK_TGZ="$TEST_ROOT_DIR/target/spark-dist-unpacked" +IMAGE_TAG_OUTPUT_FILE="$TEST_ROOT_DIR/target/image-tag.txt" +DEPLOY_MODE="minikube" +IMAGE_REPO="docker.io/kubespark" +IMAGE_TAG="N/A" +SPARK_TGZ="N/A" + +# Parse arguments +while (( "$#" )); do + case $1 in + --unpacked-spark-tgz) + UNPACKED_SPARK_TGZ="$2" + shift + ;; + --image-repo) + IMAGE_REPO="$2" + shift + ;; + --image-tag) + IMAGE_TAG="$2" + shift + ;; + --image-tag-output-file) + IMAGE_TAG_OUTPUT_FILE="$2" + shift + ;; + --deploy-mode) + DEPLOY_MODE="$2" + shift + ;; + --spark-tgz) + SPARK_TGZ="$2" + shift + ;; + *) + break + ;; + esac + shift +done + +if [[ $SPARK_TGZ == "N/A" ]]; +then + echo "Must specify a Spark tarball to build Docker images against with --spark-tgz." && exit 1; +fi + +rm -rf $UNPACKED_SPARK_TGZ +mkdir -p $UNPACKED_SPARK_TGZ +tar -xzvf $SPARK_TGZ --strip-components=1 -C $UNPACKED_SPARK_TGZ; + +if [[ $IMAGE_TAG == "N/A" ]]; +then + IMAGE_TAG=$(uuidgen); + cd $UNPACKED_SPARK_TGZ + if [[ $DEPLOY_MODE == cloud ]] ; + then + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG build + if [[ $IMAGE_REPO == gcr.io* ]] ; + then + gcloud docker -- push $IMAGE_REPO/spark:$IMAGE_TAG + else + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -r $IMAGE_REPO -t $IMAGE_TAG push + fi + else + # -m option for minikube. + $UNPACKED_SPARK_TGZ/bin/docker-image-tool.sh -m -r $IMAGE_REPO -t $IMAGE_TAG build + fi + cd - +fi + +rm -f $IMAGE_TAG_OUTPUT_FILE +echo -n $IMAGE_TAG > $IMAGE_TAG_OUTPUT_FILE diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties b/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties new file mode 100644 index 0000000000000..866126bc3c1c2 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/resources/log4j.properties @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Set everything to be logged to the file target/integration-tests.log +log4j.rootCategory=INFO, file +log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file.append=true +log4j.appender.file.file=target/integration-tests.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n + +# Ignore messages below warning level from a few verbose libraries. +log4j.logger.com.sun.jersey=WARN +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.mortbay=WARN +log4j.logger.org.spark_project.jetty=WARN diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala new file mode 100644 index 0000000000000..89c8a91c4ab58 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.File +import java.nio.file.{Path, Paths} +import java.util.UUID +import java.util.regex.Pattern + +import scala.collection.JavaConverters._ +import com.google.common.io.PatternFilenameFilter +import io.fabric8.kubernetes.api.model.{Container, Pod} +import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite} +import org.scalatest.concurrent.{Eventually, PatienceConfiguration} +import org.scalatest.time.{Minutes, Seconds, Span} + +import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} +import org.apache.spark.deploy.k8s.integrationtest.config._ + +private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll with BeforeAndAfter { + + import KubernetesSuite._ + + private var testBackend: IntegrationTestBackend = _ + private var sparkHomeDir: Path = _ + private var kubernetesTestComponents: KubernetesTestComponents = _ + private var sparkAppConf: SparkAppConf = _ + private var image: String = _ + private var containerLocalSparkDistroExamplesJar: String = _ + private var appLocator: String = _ + private var driverPodName: String = _ + + override def beforeAll(): Unit = { + // The scalatest-maven-plugin gives system properties that are referenced but not set null + // values. We need to remove the null-value properties before initializing the test backend. + val nullValueProperties = System.getProperties.asScala + .filter(entry => entry._2.equals("null")) + .map(entry => entry._1.toString) + nullValueProperties.foreach { key => + System.clearProperty(key) + } + + val sparkDirProp = System.getProperty("spark.kubernetes.test.unpackSparkDir") + require(sparkDirProp != null, "Spark home directory must be provided in system properties.") + sparkHomeDir = Paths.get(sparkDirProp) + require(sparkHomeDir.toFile.isDirectory, + s"No directory found for spark home specified at $sparkHomeDir.") + val imageTag = getTestImageTag + val imageRepo = getTestImageRepo + image = s"$imageRepo/spark:$imageTag" + + val sparkDistroExamplesJarFile: File = sparkHomeDir.resolve(Paths.get("examples", "jars")) + .toFile + .listFiles(new PatternFilenameFilter(Pattern.compile("^spark-examples_.*\\.jar$")))(0) + containerLocalSparkDistroExamplesJar = s"local:///opt/spark/examples/jars/" + + s"${sparkDistroExamplesJarFile.getName}" + testBackend = IntegrationTestBackendFactory.getTestBackend + testBackend.initialize() + kubernetesTestComponents = new KubernetesTestComponents(testBackend.getKubernetesClient) + } + + override def afterAll(): Unit = { + testBackend.cleanUp() + } + + before { + appLocator = UUID.randomUUID().toString.replaceAll("-", "") + driverPodName = "spark-test-app-" + UUID.randomUUID().toString.replaceAll("-", "") + sparkAppConf = kubernetesTestComponents.newSparkAppConf() + .set("spark.kubernetes.container.image", image) + .set("spark.kubernetes.driver.pod.name", driverPodName) + .set("spark.kubernetes.driver.label.spark-app-locator", appLocator) + .set("spark.kubernetes.executor.label.spark-app-locator", appLocator) + if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { + kubernetesTestComponents.createNamespace() + } + } + + after { + if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { + kubernetesTestComponents.deleteNamespace() + } + deleteDriverPod() + } + + test("Run SparkPi with no resources") { + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with a very long application name.") { + sparkAppConf.set("spark.app.name", "long" * 40) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with a master URL without a scheme.") { + val url = kubernetesTestComponents.kubernetesClient.getMasterUrl + val k8sMasterUrl = if (url.getPort < 0) { + s"k8s://${url.getHost}" + } else { + s"k8s://${url.getHost}:${url.getPort}" + } + sparkAppConf.set("spark.master", k8sMasterUrl) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with an argument.") { + runSparkPiAndVerifyCompletion(appArgs = Array("5")) + } + + test("Run SparkPi with custom labels, annotations, and environment variables.") { + sparkAppConf + .set("spark.kubernetes.driver.label.label1", "label1-value") + .set("spark.kubernetes.driver.label.label2", "label2-value") + .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") + .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") + .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") + .set("spark.kubernetes.executor.label.label1", "label1-value") + .set("spark.kubernetes.executor.label.label2", "label2-value") + .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") + .set("spark.executorEnv.ENV1", "VALUE1") + .set("spark.executorEnv.ENV2", "VALUE2") + + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkCustomSettings(driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkCustomSettings(executorPod) + }) + } + + test("Run SparkPi with a test secret mounted into the driver and executor pods") { + val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") + createTestSecret(secretName) + + sparkAppConf + .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + + try { + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkTestSecret(secretName, driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkTestSecret(secretName, executorPod) + }) + } finally { + deleteTestSecret(secretName) + } + } + + test("Run PageRank using remote data file") { + sparkAppConf + .set("spark.kubernetes.mountDependencies.filesDownloadDir", + CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + runSparkPageRankAndVerifyCompletion( + appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) + } + + test("Run PageRank using remote data file with test secret mounted into the driver and " + + "executors") { + val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") + createTestSecret(secretName) + + sparkAppConf + .set("spark.kubernetes.mountDependencies.filesDownloadDir", + CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) + + try { + runSparkPageRankAndVerifyCompletion( + appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkTestSecret(secretName, driverPod, withInitContainer = true) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkTestSecret(secretName, executorPod, withInitContainer = true) + }) + } finally { + deleteTestSecret(secretName) + } + } + + private def runSparkPiAndVerifyCompletion( + appResource: String = containerLocalSparkDistroExamplesJar, + driverPodChecker: Pod => Unit = doBasicDriverPodCheck, + executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, + appArgs: Array[String] = Array.empty[String], + appLocator: String = appLocator): Unit = { + runSparkApplicationAndVerifyCompletion( + appResource, + SPARK_PI_MAIN_CLASS, + Seq("Pi is roughly 3"), + appArgs, + driverPodChecker, + executorPodChecker, + appLocator) + } + + private def runSparkPageRankAndVerifyCompletion( + appResource: String = containerLocalSparkDistroExamplesJar, + driverPodChecker: Pod => Unit = doBasicDriverPodCheck, + executorPodChecker: Pod => Unit = doBasicExecutorPodCheck, + appArgs: Array[String], + appLocator: String = appLocator): Unit = { + runSparkApplicationAndVerifyCompletion( + appResource, + SPARK_PAGE_RANK_MAIN_CLASS, + Seq("1 has rank", "2 has rank", "3 has rank", "4 has rank"), + appArgs, + driverPodChecker, + executorPodChecker, + appLocator) + } + + private def runSparkApplicationAndVerifyCompletion( + appResource: String, + mainClass: String, + expectedLogOnCompletion: Seq[String], + appArgs: Array[String], + driverPodChecker: Pod => Unit, + executorPodChecker: Pod => Unit, + appLocator: String): Unit = { + val appArguments = SparkAppArguments( + mainAppResource = appResource, + mainClass = mainClass, + appArgs = appArgs) + SparkAppLauncher.launch(appArguments, sparkAppConf, TIMEOUT.value.toSeconds.toInt, sparkHomeDir) + + val driverPod = kubernetesTestComponents.kubernetesClient + .pods() + .withLabel("spark-app-locator", appLocator) + .withLabel("spark-role", "driver") + .list() + .getItems + .get(0) + driverPodChecker(driverPod) + + val executorPods = kubernetesTestComponents.kubernetesClient + .pods() + .withLabel("spark-app-locator", appLocator) + .withLabel("spark-role", "executor") + .list() + .getItems + executorPods.asScala.foreach { pod => + executorPodChecker(pod) + } + + Eventually.eventually(TIMEOUT, INTERVAL) { + expectedLogOnCompletion.foreach { e => + assert(kubernetesTestComponents.kubernetesClient + .pods() + .withName(driverPod.getMetadata.getName) + .getLog + .contains(e), "The application did not complete.") + } + } + } + + private def doBasicDriverPodCheck(driverPod: Pod): Unit = { + assert(driverPod.getMetadata.getName === driverPodName) + assert(driverPod.getSpec.getContainers.get(0).getImage === image) + assert(driverPod.getSpec.getContainers.get(0).getName === "spark-kubernetes-driver") + } + + private def doBasicExecutorPodCheck(executorPod: Pod): Unit = { + assert(executorPod.getSpec.getContainers.get(0).getImage === image) + assert(executorPod.getSpec.getContainers.get(0).getName === "executor") + } + + private def checkCustomSettings(pod: Pod): Unit = { + assert(pod.getMetadata.getLabels.get("label1") === "label1-value") + assert(pod.getMetadata.getLabels.get("label2") === "label2-value") + assert(pod.getMetadata.getAnnotations.get("annotation1") === "annotation1-value") + assert(pod.getMetadata.getAnnotations.get("annotation2") === "annotation2-value") + + val container = pod.getSpec.getContainers.get(0) + val envVars = container + .getEnv + .asScala + .map { env => + (env.getName, env.getValue) + } + .toMap + assert(envVars("ENV1") === "VALUE1") + assert(envVars("ENV2") === "VALUE2") + } + + private def deleteDriverPod(): Unit = { + kubernetesTestComponents.kubernetesClient.pods().withName(driverPodName).delete() + Eventually.eventually(TIMEOUT, INTERVAL) { + assert(kubernetesTestComponents.kubernetesClient + .pods() + .withName(driverPodName) + .get() == null) + } + } + + private def createTestSecret(secretName: String): Unit = { + kubernetesTestComponents.kubernetesClient.secrets + .createNew() + .editOrNewMetadata() + .withName(secretName) + .endMetadata() + .addToStringData(TEST_SECRET_KEY, TEST_SECRET_VALUE) + .done() + } + + private def checkTestSecret( + secretName: String, + pod: Pod, + withInitContainer: Boolean = false): Unit = { + val testSecretVolume = pod.getSpec.getVolumes.asScala.filter { volume => + volume.getName == s"$secretName-volume" + } + assert(testSecretVolume.size === 1) + assert(testSecretVolume.head.getSecret.getSecretName === secretName) + + checkTestSecretInContainer(secretName, pod.getSpec.getContainers.get(0)) + + if (withInitContainer) { + checkTestSecretInContainer(secretName, pod.getSpec.getInitContainers.get(0)) + } + } + + private def checkTestSecretInContainer(secretName: String, container: Container): Unit = { + val testSecret = container.getVolumeMounts.asScala.filter { mount => + mount.getName == s"$secretName-volume" + } + assert(testSecret.size === 1) + assert(testSecret.head.getMountPath === TEST_SECRET_MOUNT_PATH) + } + + private def deleteTestSecret(secretName: String): Unit = { + kubernetesTestComponents.kubernetesClient.secrets + .withName(secretName) + .delete() + + Eventually.eventually(TIMEOUT, INTERVAL) { + assert(kubernetesTestComponents.kubernetesClient.secrets.withName(secretName).get() == null) + } + } +} + +private[spark] object KubernetesSuite { + + val TIMEOUT = PatienceConfiguration.Timeout(Span(2, Minutes)) + val INTERVAL = PatienceConfiguration.Interval(Span(2, Seconds)) + val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" + val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" + + val TEST_SECRET_NAME_PREFIX = "test-secret-" + val TEST_SECRET_KEY = "test-key" + val TEST_SECRET_VALUE = "test-data" + val TEST_SECRET_MOUNT_PATH = "/etc/secrets" + + val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" + + val REMOTE_PAGE_RANK_DATA_FILE = + "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" + val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = + s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" + + case object ShuffleNotReadyException extends Exception +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala new file mode 100644 index 0000000000000..b9c87d59fe27b --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.nio.file.{Path, Paths} +import java.util.UUID + +import scala.collection.mutable +import scala.collection.JavaConverters._ +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import org.scalatest.concurrent.Eventually + +private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) { + + val namespaceOption = Option(System.getProperty("spark.kubernetes.test.namespace")) + val hasUserSpecifiedNamespace = namespaceOption.isDefined + val namespace = namespaceOption.getOrElse(UUID.randomUUID().toString.replaceAll("-", "")) + private val serviceAccountName = + Option(System.getProperty("spark.kubernetes.test.serviceAccountName")) + .getOrElse("default") + val kubernetesClient = defaultClient.inNamespace(namespace) + val clientConfig = kubernetesClient.getConfiguration + + def createNamespace(): Unit = { + defaultClient.namespaces.createNew() + .withNewMetadata() + .withName(namespace) + .endMetadata() + .done() + } + + def deleteNamespace(): Unit = { + defaultClient.namespaces.withName(namespace).delete() + Eventually.eventually(KubernetesSuite.TIMEOUT, KubernetesSuite.INTERVAL) { + val namespaceList = defaultClient + .namespaces() + .list() + .getItems + .asScala + require(!namespaceList.exists(_.getMetadata.getName == namespace)) + } + } + + def newSparkAppConf(): SparkAppConf = { + new SparkAppConf() + .set("spark.master", s"k8s://${kubernetesClient.getMasterUrl}") + .set("spark.kubernetes.namespace", namespace) + .set("spark.executor.memory", "500m") + .set("spark.executor.cores", "1") + .set("spark.executors.instances", "1") + .set("spark.app.name", "spark-test-app") + .set("spark.ui.enabled", "true") + .set("spark.testing", "false") + .set("spark.kubernetes.submission.waitAppCompletion", "false") + .set("spark.kubernetes.authenticate.driver.serviceAccountName", serviceAccountName) + } +} + +private[spark] class SparkAppConf { + + private val map = mutable.Map[String, String]() + + def set(key: String, value: String): SparkAppConf = { + map.put(key, value) + this + } + + def get(key: String): String = map.getOrElse(key, "") + + def setJars(jars: Seq[String]): Unit = set("spark.jars", jars.mkString(",")) + + override def toString: String = map.toString + + def toStringArray: Iterable[String] = map.toList.flatMap(t => List("--conf", s"${t._1}=${t._2}")) +} + +private[spark] case class SparkAppArguments( + mainAppResource: String, + mainClass: String, + appArgs: Array[String]) + +private[spark] object SparkAppLauncher extends Logging { + + def launch( + appArguments: SparkAppArguments, + appConf: SparkAppConf, + timeoutSecs: Int, + sparkHomeDir: Path): Unit = { + val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) + logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") + val commandLine = Array(sparkSubmitExecutable.toFile.getAbsolutePath, + "--deploy-mode", "cluster", + "--class", appArguments.mainClass, + "--master", appConf.get("spark.master") + ) ++ appConf.toStringArray :+ + appArguments.mainAppResource :+ + appArguments.appArgs.mkString(" ") + logInfo(s"Launching a spark app with command line: ${commandLine.mkString(" ")}") + ProcessUtils.executeProcess(commandLine, timeoutSecs) + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala new file mode 100644 index 0000000000000..459c0a4138b86 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import org.apache.log4j.{Logger, LogManager, Priority} + +trait Logging { + + private val log: Logger = LogManager.getLogger(this.getClass) + + protected def logDebug(msg: => String) = if (log.isDebugEnabled) log.debug(msg) + + protected def logInfo(msg: => String) = if (log.isInfoEnabled) log.info(msg) + + protected def logWarning(msg: => String) = if (log.isEnabledFor(Priority.WARN)) log.warn(msg) + + protected def logWarning(msg: => String, throwable: Throwable) = + if (log.isEnabledFor(Priority.WARN)) log.warn(msg, throwable) + + protected def logError(msg: => String) = if (log.isEnabledFor(Priority.ERROR)) log.error(msg) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala new file mode 100644 index 0000000000000..aa6425ddd0353 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.util.concurrent.TimeUnit + +import scala.collection.mutable.ArrayBuffer +import scala.io.Source + +object ProcessUtils extends Logging { + /** + * executeProcess is used to run a command and return the output if it + * completes within timeout seconds. + */ + def executeProcess(fullCommand: Array[String], timeout: Long): Seq[String] = { + val pb = new ProcessBuilder().command(fullCommand: _*) + pb.redirectErrorStream(true) + val proc = pb.start() + val outputLines = new ArrayBuffer[String] + Utils.tryWithResource(proc.getInputStream)( + Source.fromInputStream(_, "UTF-8").getLines().foreach { line => + logInfo(line) + outputLines += line + }) + assert(proc.waitFor(timeout, TimeUnit.SECONDS), + s"Timed out while executing ${fullCommand.mkString(" ")}") + assert(proc.exitValue == 0, s"Failed to execute ${fullCommand.mkString(" ")}") + outputLines + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala new file mode 100644 index 0000000000000..f1fd6dc19ce54 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/SparkReadinessWatcher.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.util.concurrent.TimeUnit + +import com.google.common.util.concurrent.SettableFuture +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.client.{KubernetesClientException, Watcher} +import io.fabric8.kubernetes.client.Watcher.Action +import io.fabric8.kubernetes.client.internal.readiness.Readiness + +private[spark] class SparkReadinessWatcher[T <: HasMetadata] extends Watcher[T] { + + private val signal = SettableFuture.create[Boolean] + + override def eventReceived(action: Action, resource: T): Unit = { + if ((action == Action.MODIFIED || action == Action.ADDED) && + Readiness.isReady(resource)) { + signal.set(true) + } + } + + override def onClose(cause: KubernetesClientException): Unit = {} + + def waitUntilReady(): Boolean = signal.get(60, TimeUnit.SECONDS) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala new file mode 100644 index 0000000000000..c300ca46083d4 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.Closeable +import java.net.URI + +object Utils extends Logging { + + def tryWithResource[R <: Closeable, T](createResource: => R)(f: R => T): T = { + val resource = createResource + try f.apply(resource) finally resource.close() + } + + def tryWithSafeFinally[T](block: => T)(finallyBlock: => Unit): T = { + var originalThrowable: Throwable = null + try { + block + } catch { + case t: Throwable => + // Purposefully not using NonFatal, because even fatal exceptions + // we don't want to have our finallyBlock suppress + originalThrowable = t + throw originalThrowable + } finally { + try { + finallyBlock + } catch { + case t: Throwable => + if (originalThrowable != null) { + originalThrowable.addSuppressed(t) + logWarning(s"Suppressing exception in finally: " + t.getMessage, t) + throw originalThrowable + } else { + throw t + } + } + } + } + + def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { + require(rawMasterURL.startsWith("k8s://"), + "Kubernetes master URL must start with k8s://.") + val masterWithoutK8sPrefix = rawMasterURL.substring("k8s://".length) + + // To handle master URLs, e.g., k8s://host:port. + if (!masterWithoutK8sPrefix.contains("://")) { + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + return s"k8s://$resolvedURL" + } + + val masterScheme = new URI(masterWithoutK8sPrefix).getScheme + val resolvedURL = masterScheme.toLowerCase match { + case "https" => + masterWithoutK8sPrefix + case "http" => + logWarning("Kubernetes master URL uses HTTP instead of HTTPS.") + masterWithoutK8sPrefix + case null => + val resolvedURL = s"https://$masterWithoutK8sPrefix" + logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + + s"URL is $resolvedURL.") + resolvedURL + case _ => + throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) + } + + s"k8s://$resolvedURL" + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala new file mode 100644 index 0000000000000..00b927ff7c327 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.k8s.integrationtest.backend + +import io.fabric8.kubernetes.client.DefaultKubernetesClient + +import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.CloudTestBackend +import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend + +private[spark] trait IntegrationTestBackend { + def initialize(): Unit + def getKubernetesClient: DefaultKubernetesClient + def cleanUp(): Unit = {} +} + +private[spark] object IntegrationTestBackendFactory { + def getTestBackend: IntegrationTestBackend = { + val deployMode = Option(System.getProperty("spark.kubernetes.test.deployMode")) + .getOrElse("minikube") + if (deployMode == "minikube") { + MinikubeTestBackend + } else { + CloudTestBackend + } + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala new file mode 100644 index 0000000000000..2f2aeada12a5b --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.cloud + +import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} + +import org.apache.spark.deploy.k8s.integrationtest.Utils +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object CloudTestBackend extends IntegrationTestBackend { + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + val masterUrl = Option(System.getProperty("spark.kubernetes.test.master")) + .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) + val k8sConf = new ConfigBuilder() + .withApiVersion("v1") + .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) + .build() + defaultClient = new DefaultKubernetesClient(k8sConf) + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala new file mode 100644 index 0000000000000..7145d85fb5a1a --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.minikube + +import java.io.File +import java.nio.file.Paths + +import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} + +import org.apache.spark.deploy.k8s.integrationtest.{Logging, ProcessUtils} + +// TODO support windows +private[spark] object Minikube extends Logging { + + private val MINIKUBE_STARTUP_TIMEOUT_SECONDS = 60 + + def getMinikubeIp: String = { + val outputs = executeMinikube("ip") + .filter(_.matches("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$")) + assert(outputs.size == 1, "Unexpected amount of output from minikube ip") + outputs.head + } + + def getMinikubeStatus: MinikubeStatus.Value = { + val statusString = executeMinikube("status") + .filter(line => line.contains("minikubeVM: ") || line.contains("minikube:")) + .head + .replaceFirst("minikubeVM: ", "") + .replaceFirst("minikube: ", "") + MinikubeStatus.unapply(statusString) + .getOrElse(throw new IllegalStateException(s"Unknown status $statusString")) + } + + def getKubernetesClient: DefaultKubernetesClient = { + val kubernetesMaster = s"https://${getMinikubeIp}:8443" + val userHome = System.getProperty("user.home") + val kubernetesConf = new ConfigBuilder() + .withApiVersion("v1") + .withMasterUrl(kubernetesMaster) + .withCaCertFile(Paths.get(userHome, ".minikube", "ca.crt").toFile.getAbsolutePath) + .withClientCertFile(Paths.get(userHome, ".minikube", "apiserver.crt").toFile.getAbsolutePath) + .withClientKeyFile(Paths.get(userHome, ".minikube", "apiserver.key").toFile.getAbsolutePath) + .build() + new DefaultKubernetesClient(kubernetesConf) + } + + private def executeMinikube(action: String, args: String*): Seq[String] = { + ProcessUtils.executeProcess( + Array("bash", "-c", s"minikube $action") ++ args, MINIKUBE_STARTUP_TIMEOUT_SECONDS) + } +} + +private[spark] object MinikubeStatus extends Enumeration { + + // The following states are listed according to + // https://github.com/docker/machine/blob/master/libmachine/state/state.go. + val STARTING = status("Starting") + val RUNNING = status("Running") + val PAUSED = status("Paused") + val STOPPING = status("Stopping") + val STOPPED = status("Stopped") + val ERROR = status("Error") + val TIMEOUT = status("Timeout") + val SAVED = status("Saved") + val NONE = status("") + + def status(value: String): Value = new Val(nextId, value) + def unapply(s: String): Option[Value] = values.find(s == _.toString) +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala new file mode 100644 index 0000000000000..cb9324179d70e --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/MinikubeTestBackend.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest.backend.minikube + +import io.fabric8.kubernetes.client.DefaultKubernetesClient + +import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend + +private[spark] object MinikubeTestBackend extends IntegrationTestBackend { + + private var defaultClient: DefaultKubernetesClient = _ + + override def initialize(): Unit = { + val minikubeStatus = Minikube.getMinikubeStatus + require(minikubeStatus == MinikubeStatus.RUNNING, + s"Minikube must be running to use the Minikube backend for integration tests." + + s" Current status is: $minikubeStatus.") + defaultClient = Minikube.getKubernetesClient + } + + override def cleanUp(): Unit = { + super.cleanUp() + } + + override def getKubernetesClient: DefaultKubernetesClient = { + defaultClient + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala new file mode 100644 index 0000000000000..a81ef455c6766 --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/config.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +import java.io.File + +import com.google.common.base.Charsets +import com.google.common.io.Files + +package object config { + def getTestImageTag: String = { + val imageTagFileProp = System.getProperty("spark.kubernetes.test.imageTagFile") + require(imageTagFileProp != null, "Image tag file must be provided in system properties.") + val imageTagFile = new File(imageTagFileProp) + require(imageTagFile.isFile, s"No file found for image tag at ${imageTagFile.getAbsolutePath}.") + Files.toString(imageTagFile, Charsets.UTF_8).trim + } + + def getTestImageRepo: String = { + val imageRepo = System.getProperty("spark.kubernetes.test.imageRepo") + require(imageRepo != null, "Image repo must be provided in system properties.") + imageRepo + } +} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala new file mode 100644 index 0000000000000..0807a68cd823c --- /dev/null +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/constants.scala @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.deploy.k8s.integrationtest + +package object constants { + val MINIKUBE_TEST_BACKEND = "minikube" + val GCE_TEST_BACKEND = "gce" +} From 81c7a66ad68d75b9f1f158f375f89ca8926ae17a Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 09:34:56 -0800 Subject: [PATCH 15/39] Make k8s integration tests build when top-level kubernetes profile selected --- pom.xml | 1 + resource-managers/kubernetes/integration-tests/pom.xml | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 883c096ae1ae9..23bbd3b09734e 100644 --- a/pom.xml +++ b/pom.xml @@ -2705,6 +2705,7 @@ kubernetes resource-managers/kubernetes/core + resource-managers/kubernetes/integration-tests diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index e3e48fcc07f7d..f1bfada70b473 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -17,6 +17,12 @@ --> 4.0.0 + + org.apache.spark + spark-parent_2.11 + 2.4.0-SNAPSHOT + ../../../pom.xml + spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests @@ -97,7 +103,6 @@ org.scalatest scalatest_${scala.binary.version} - ${scalatest.version} test From bef586f74096f0e67b40044ea0317c4406f11cac Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 10:15:07 -0800 Subject: [PATCH 16/39] Remove LICENSE and copy of mvn wrapper script. Rewrite path for calling mvn wrapper script. --- .../kubernetes/integration-tests/LICENSE | 201 ------------------ .../kubernetes/integration-tests/build/mvn | 29 --- .../dev/dev-run-integration-tests.sh | 2 +- 3 files changed, 1 insertion(+), 231 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/LICENSE delete mode 100755 resource-managers/kubernetes/integration-tests/build/mvn diff --git a/resource-managers/kubernetes/integration-tests/LICENSE b/resource-managers/kubernetes/integration-tests/LICENSE deleted file mode 100644 index 261eeb9e9f8b2..0000000000000 --- a/resource-managers/kubernetes/integration-tests/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/resource-managers/kubernetes/integration-tests/build/mvn b/resource-managers/kubernetes/integration-tests/build/mvn deleted file mode 100755 index 87e5b58640c5c..0000000000000 --- a/resource-managers/kubernetes/integration-tests/build/mvn +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -BUILD_DIR=$(dirname $0) - -MVN_RUNNER=$BUILD_DIR/run-mvn - -if [ ! -f $MVN_RUNNER ]; -then - curl -s --progress-bar https://raw.githubusercontent.com/apache/spark/master/build/mvn > $MVN_RUNNER - chmod +x $MVN_RUNNER -fi -source $MVN_RUNNER diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 33879d5fc8b2a..eb9236981cfcf 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -107,4 +107,4 @@ then properties=( ${properties[@]} -Dspark.kubernetes.test.master=$SPARK_MASTER ) fi -build/mvn integration-test ${properties[@]} +../../../build/mvn integration-test ${properties[@]} From 3615953bea1b6921802940334f44d3cece7c709b Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 11:13:00 -0800 Subject: [PATCH 17/39] Fix scala style issues --- .../k8s/integrationtest/KubernetesSuite.scala | 215 +++++++++--------- .../KubernetesTestComponents.scala | 2 +- 2 files changed, 109 insertions(+), 108 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 89c8a91c4ab58..6203e29785820 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -22,16 +22,17 @@ import java.util.UUID import java.util.regex.Pattern import scala.collection.JavaConverters._ + import com.google.common.io.PatternFilenameFilter import io.fabric8.kubernetes.api.model.{Container, Pod} -import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSuite} +import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.{Minutes, Seconds, Span} import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} import org.apache.spark.deploy.k8s.integrationtest.config._ -private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll with BeforeAndAfter { +private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter { import KubernetesSuite._ @@ -101,111 +102,111 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit runSparkPiAndVerifyCompletion() } - test("Run SparkPi with a very long application name.") { - sparkAppConf.set("spark.app.name", "long" * 40) - runSparkPiAndVerifyCompletion() - } - - test("Run SparkPi with a master URL without a scheme.") { - val url = kubernetesTestComponents.kubernetesClient.getMasterUrl - val k8sMasterUrl = if (url.getPort < 0) { - s"k8s://${url.getHost}" - } else { - s"k8s://${url.getHost}:${url.getPort}" - } - sparkAppConf.set("spark.master", k8sMasterUrl) - runSparkPiAndVerifyCompletion() - } - - test("Run SparkPi with an argument.") { - runSparkPiAndVerifyCompletion(appArgs = Array("5")) - } - - test("Run SparkPi with custom labels, annotations, and environment variables.") { - sparkAppConf - .set("spark.kubernetes.driver.label.label1", "label1-value") - .set("spark.kubernetes.driver.label.label2", "label2-value") - .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") - .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") - .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") - .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") - .set("spark.kubernetes.executor.label.label1", "label1-value") - .set("spark.kubernetes.executor.label.label2", "label2-value") - .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") - .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") - .set("spark.executorEnv.ENV1", "VALUE1") - .set("spark.executorEnv.ENV2", "VALUE2") - - runSparkPiAndVerifyCompletion( - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkCustomSettings(driverPod) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkCustomSettings(executorPod) - }) - } - - test("Run SparkPi with a test secret mounted into the driver and executor pods") { - val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") - createTestSecret(secretName) - - sparkAppConf - .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - - try { - runSparkPiAndVerifyCompletion( - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkTestSecret(secretName, driverPod) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkTestSecret(secretName, executorPod) - }) - } finally { - deleteTestSecret(secretName) - } - } - - test("Run PageRank using remote data file") { - sparkAppConf - .set("spark.kubernetes.mountDependencies.filesDownloadDir", - CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) - .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - runSparkPageRankAndVerifyCompletion( - appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) - } - - test("Run PageRank using remote data file with test secret mounted into the driver and " + - "executors") { - val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") - createTestSecret(secretName) - - sparkAppConf - .set("spark.kubernetes.mountDependencies.filesDownloadDir", - CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) - .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) - - try { - runSparkPageRankAndVerifyCompletion( - appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), - driverPodChecker = (driverPod: Pod) => { - doBasicDriverPodCheck(driverPod) - checkTestSecret(secretName, driverPod, withInitContainer = true) - }, - executorPodChecker = (executorPod: Pod) => { - doBasicExecutorPodCheck(executorPod) - checkTestSecret(secretName, executorPod, withInitContainer = true) - }) - } finally { - deleteTestSecret(secretName) - } - } +// test("Run SparkPi with a very long application name.") { +// sparkAppConf.set("spark.app.name", "long" * 40) +// runSparkPiAndVerifyCompletion() +// } +// +// test("Run SparkPi with a master URL without a scheme.") { +// val url = kubernetesTestComponents.kubernetesClient.getMasterUrl +// val k8sMasterUrl = if (url.getPort < 0) { +// s"k8s://${url.getHost}" +// } else { +// s"k8s://${url.getHost}:${url.getPort}" +// } +// sparkAppConf.set("spark.master", k8sMasterUrl) +// runSparkPiAndVerifyCompletion() +// } +// +// test("Run SparkPi with an argument.") { +// runSparkPiAndVerifyCompletion(appArgs = Array("5")) +// } +// +// test("Run SparkPi with custom labels, annotations, and environment variables.") { +// sparkAppConf +// .set("spark.kubernetes.driver.label.label1", "label1-value") +// .set("spark.kubernetes.driver.label.label2", "label2-value") +// .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") +// .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") +// .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") +// .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") +// .set("spark.kubernetes.executor.label.label1", "label1-value") +// .set("spark.kubernetes.executor.label.label2", "label2-value") +// .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") +// .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") +// .set("spark.executorEnv.ENV1", "VALUE1") +// .set("spark.executorEnv.ENV2", "VALUE2") +// +// runSparkPiAndVerifyCompletion( +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkCustomSettings(driverPod) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkCustomSettings(executorPod) +// }) +// } +// +// test("Run SparkPi with a test secret mounted into the driver and executor pods") { +// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") +// createTestSecret(secretName) +// +// sparkAppConf +// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// +// try { +// runSparkPiAndVerifyCompletion( +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkTestSecret(secretName, driverPod) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkTestSecret(secretName, executorPod) +// }) +// } finally { +// deleteTestSecret(secretName) +// } +// } +// +// test("Run PageRank using remote data file") { +// sparkAppConf +// .set("spark.kubernetes.mountDependencies.filesDownloadDir", +// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) +// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) +// runSparkPageRankAndVerifyCompletion( +// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) +// } +// +// test("Run PageRank using remote data file with test secret mounted into the driver and " + +// "executors") { +// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") +// createTestSecret(secretName) +// +// sparkAppConf +// .set("spark.kubernetes.mountDependencies.filesDownloadDir", +// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) +// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) +// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) +// +// try { +// runSparkPageRankAndVerifyCompletion( +// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), +// driverPodChecker = (driverPod: Pod) => { +// doBasicDriverPodCheck(driverPod) +// checkTestSecret(secretName, driverPod, withInitContainer = true) +// }, +// executorPodChecker = (executorPod: Pod) => { +// doBasicExecutorPodCheck(executorPod) +// checkTestSecret(secretName, executorPod, withInitContainer = true) +// }) +// } finally { +// deleteTestSecret(secretName) +// } +// } private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index b9c87d59fe27b..576bf6a2c5d8b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -19,8 +19,8 @@ package org.apache.spark.deploy.k8s.integrationtest import java.nio.file.{Path, Paths} import java.util.UUID -import scala.collection.mutable import scala.collection.JavaConverters._ +import scala.collection.mutable import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually From f29679ef5666a23b9451ca22b7897a4c03a9a26e Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 13:41:45 -0800 Subject: [PATCH 18/39] Ignore dist/ for style checks --- dev/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tox.ini b/dev/tox.ini index 583c1eaaa966b..28dad8f3b5c7c 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -16,4 +16,4 @@ [pycodestyle] ignore=E402,E731,E241,W503,E226,E722,E741,E305 max-line-length=100 -exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/* +exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/*,dist/* From 9b0eede2449a40a35c1793ef0051bc8f9275ec37 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 7 Mar 2018 13:42:55 -0800 Subject: [PATCH 19/39] Fixes for scala style --- .../kubernetes/integration-tests/pom.xml | 15 ++++++++++++++- .../k8s/integrationtest/KubernetesSuite.scala | 4 +++- .../KubernetesTestComponents.scala | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index f1bfada70b473..c3d06dadebe20 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -26,7 +26,6 @@ spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests - 0.1-SNAPSHOT 3.3.9 3.5 @@ -57,6 +56,20 @@ Spark Project Kubernetes Integration Tests + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + + + + org.apache.spark + spark-core_${scala.binary.version} + ${project.version} + test-jar + test + + commons-logging commons-logging diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 6203e29785820..7353e82569c57 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -29,10 +29,12 @@ import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll} import org.scalatest.concurrent.{Eventually, PatienceConfiguration} import org.scalatest.time.{Minutes, Seconds, Span} +import org.apache.spark.SparkFunSuite import org.apache.spark.deploy.k8s.integrationtest.backend.{IntegrationTestBackend, IntegrationTestBackendFactory} import org.apache.spark.deploy.k8s.integrationtest.config._ -private[spark] class KubernetesSuite extends SparkFunSuite with BeforeAndAfterAll with BeforeAndAfter { +private[spark] class KubernetesSuite extends SparkFunSuite + with BeforeAndAfterAll with BeforeAndAfter { import KubernetesSuite._ diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 576bf6a2c5d8b..2df59b6a3373e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -21,6 +21,7 @@ import java.util.UUID import scala.collection.JavaConverters._ import scala.collection.mutable + import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually From e55b8a723ee005af16ecd70d2be55986c361f758 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:08:37 -0700 Subject: [PATCH 20/39] Remove repository cloning behavior and allow script to be called from other directories --- .../integration-tests/dev/dev-run-integration-tests.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index eb9236981cfcf..2b47a42890c38 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -16,10 +16,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # +TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests + +cd "${TEST_ROOT_DIR}" source ./include/util.sh -TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/integration-tests BRANCH="master" SPARK_REPO="https://github.com/apache/spark" SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" @@ -77,12 +79,6 @@ while (( "$#" )); do shift done -if [[ $SPARK_TGZ == "N/A" ]]; -then - echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." - clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH -fi - cd $TEST_ROOT_DIR properties=( From a0023b2f337ee45c46c84ba5e8fad4c6dbbfd952 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:09:54 -0700 Subject: [PATCH 21/39] Remove config options that were only used during repo clone process --- .../dev/dev-run-integration-tests.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 2b47a42890c38..754588194d607 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -22,9 +22,6 @@ cd "${TEST_ROOT_DIR}" source ./include/util.sh -BRANCH="master" -SPARK_REPO="https://github.com/apache/spark" -SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" DEPLOY_MODE="minikube" IMAGE_REPO="docker.io/kubespark" SPARK_TGZ="N/A" @@ -36,14 +33,6 @@ SERVICE_ACCOUNT= # Parse arguments while (( "$#" )); do case $1 in - --spark-branch) - BRANCH="$2" - shift - ;; - --spark-repo) - SPARK_REPO="$2" - shift - ;; --image-repo) IMAGE_REPO="$2" shift From e70f3bea3d5e5e21711edff432c52ce80e5c4424 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 14:52:35 -0700 Subject: [PATCH 22/39] Remove K8s cloud-based backend testing support from this PR --- .../backend/IntegrationTestBackend.scala | 8 ++-- .../backend/cloud/CloudTestBackend.scala | 41 ------------------- 2 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index 00b927ff7c327..e9a202677654d 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -19,7 +19,6 @@ package org.apache.spark.deploy.k8s.integrationtest.backend import io.fabric8.kubernetes.client.DefaultKubernetesClient -import org.apache.spark.deploy.k8s.integrationtest.backend.cloud.CloudTestBackend import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.MinikubeTestBackend private[spark] trait IntegrationTestBackend { @@ -29,13 +28,16 @@ private[spark] trait IntegrationTestBackend { } private[spark] object IntegrationTestBackendFactory { + val DeployModeConfigKey = "spark.kubernetes.test.deployMode" + def getTestBackend: IntegrationTestBackend = { - val deployMode = Option(System.getProperty("spark.kubernetes.test.deployMode")) + val deployMode = Option(System.getProperty(DeployModeConfigKey)) .getOrElse("minikube") if (deployMode == "minikube") { MinikubeTestBackend } else { - CloudTestBackend + throw new IllegalArgumentException( + "Invalid " + DeployModeConfigKey + ": " + deployMode) } } } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala deleted file mode 100644 index 2f2aeada12a5b..0000000000000 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/cloud/CloudTestBackend.scala +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.deploy.k8s.integrationtest.backend.cloud - -import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} - -import org.apache.spark.deploy.k8s.integrationtest.Utils -import org.apache.spark.deploy.k8s.integrationtest.backend.IntegrationTestBackend - -private[spark] object CloudTestBackend extends IntegrationTestBackend { - - private var defaultClient: DefaultKubernetesClient = _ - - override def initialize(): Unit = { - val masterUrl = Option(System.getProperty("spark.kubernetes.test.master")) - .getOrElse(throw new RuntimeException("Kubernetes master URL is not set")) - val k8sConf = new ConfigBuilder() - .withApiVersion("v1") - .withMasterUrl(Utils.checkAndGetK8sMasterUrl(masterUrl).replaceFirst("k8s://", "")) - .build() - defaultClient = new DefaultKubernetesClient(k8sConf) - } - - override def getKubernetesClient: DefaultKubernetesClient = { - defaultClient - } -} From e6bd56325dbf9bdeda9219fd8d40b40db4785ee5 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:12:08 -0700 Subject: [PATCH 23/39] Update README.md excluding cloning and building logic --- .../kubernetes/integration-tests/README.md | 38 ++----------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 97cb8ab42e7a0..b3863e6b7d1af 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -23,21 +23,6 @@ You can download Minikube [here](https://github.com/kubernetes/minikube/releases Configuration of the integration test runtime is done through passing different arguments to the test script. The main useful options are outlined below. -## Use a non-local cluster - -To use your own cluster running in the cloud, set the following: - -* `--deploy-mode cloud` to indicate that the test is connecting to a remote cluster instead of Minikube, -* `--spark-master ` - set `` to the externally accessible Kubernetes cluster URL, -* `--image-repo ` - set `` to a write-accessible Docker image repository that provides the images for your cluster. The framework assumes your local Docker client can push to this repository. - -Therefore the command looks like this: - - dev/dev-run-integration-tests.sh \ - --deploy-mode cloud \ - --spark-master https://example.com:8443/apiserver \ - --image-repo docker.example.com/spark-images - ## Re-using Docker Images By default, the test framework will build new Docker images on every test execution. A unique image tag is generated, @@ -50,28 +35,13 @@ where if you still want to use images that were built before by the test framewo dev/dev-run-integration-tests.sh --image-tag $(cat target/imageTag.txt) -## Customizing the Spark Source Code to Test - -By default, the test framework will test the master branch of Spark from [here](https://github.com/apache/spark). You -can specify the following options to test against different source versions of Spark: - -* `--spark-repo ` - set `` to the git or http URI of the Spark git repository to clone -* `--spark-branch ` - set `` to the branch of the repository to build. - - -An example: - - dev/dev-run-integration-tests.sh \ - --spark-repo https://github.com/apache-spark-on-k8s/spark \ - --spark-branch new-feature +## Spark Distribution Under Test -Additionally, you can use a pre-built Spark distribution. In this case, the repository is not cloned at all, and no -source code has to be compiled. +The Spark code to test is handed to the integration test system via a tarball. Here is the option that is used to specify the tarball: * `--spark-tgz ` - set `` to point to a tarball containing the Spark distribution to test. -When the tests are cloning a repository and building it, the Spark distribution is placed in `target/spark/spark-.tgz`. -Reuse this tarball to save a significant amount of time if you are iterating on the development of these integration tests. +TODO: Don't require the packaging of the built Spark artifacts into this tarball, just read them out of the current tree. ## Customizing the Namespace and Service Account @@ -79,4 +49,4 @@ Reuse this tarball to save a significant amount of time if you are iterating on * `--service-account ` - set `` to the name of the Kubernetes service account to use in the namespace specified by the `--namespace`. The service account is expected to have permissions to get, list, watch, and create pods. For clusters with RBAC turned on, it's important that the right permissions are granted to the service account -in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. \ No newline at end of file +in the namespace through an appropriate role and role binding. A reference RBAC configuration is provided in `dev/spark-rbac.yaml`. From 9e64f43b62ea4f4df351e61f102be83309a08f36 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:13:17 -0700 Subject: [PATCH 24/39] Remove unnecessary cloning and building code for the Spark repo --- .../dev/dev-run-integration-tests.sh | 2 - .../integration-tests/include/util.sh | 43 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/include/util.sh diff --git a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh index 754588194d607..ea893fa39eede 100755 --- a/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh +++ b/resource-managers/kubernetes/integration-tests/dev/dev-run-integration-tests.sh @@ -20,8 +20,6 @@ TEST_ROOT_DIR=$(git rev-parse --show-toplevel)/resource-managers/kubernetes/inte cd "${TEST_ROOT_DIR}" -source ./include/util.sh - DEPLOY_MODE="minikube" IMAGE_REPO="docker.io/kubespark" SPARK_TGZ="N/A" diff --git a/resource-managers/kubernetes/integration-tests/include/util.sh b/resource-managers/kubernetes/integration-tests/include/util.sh deleted file mode 100644 index 49b9114854120..0000000000000 --- a/resource-managers/kubernetes/integration-tests/include/util.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -clone_build_spark() { - spark_repo=$1 - spark_repo_local_dir=$2 - branch=$3 - pushd . - - # clone spark distribution if needed. - # TODO(ssuchter): This code, that does a checkout of a spark repo, - # made more sense when this script was in the repo - # https://github.com/apache-spark-on-k8s/spark-integration - # but now we shouldn't check out another copy of spark, we should just - # build in the copy that is checked out already. - if [ -d "$spark_repo_local_dir" ]; - then - (cd $spark_repo_local_dir && git fetch origin $branch); - else - mkdir -p $spark_repo_local_dir; - git clone -b $branch --single-branch $spark_repo $spark_repo_local_dir; - fi - cd $spark_repo_local_dir - git checkout -B $branch origin/$branch - ./dev/make-distribution.sh --tgz -Phadoop-2.7 -Pkubernetes -DskipTests; - SPARK_TGZ=$(find $spark_repo_local_dir -name spark-*.tgz) - - popd -} From 3ba6ffb5f2d17f984380cb5bca1f091be8a72b49 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:17:46 -0700 Subject: [PATCH 25/39] Remove e2e-prow.sh, which isn't appropriate for this PR --- .../integration-tests/e2e/e2e-prow.sh | 69 ------------------- 1 file changed, 69 deletions(-) delete mode 100755 resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh diff --git a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh b/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh deleted file mode 100755 index 501f350943e73..0000000000000 --- a/resource-managers/kubernetes/integration-tests/e2e/e2e-prow.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -### This script is used by Kubernetes Test Infrastructure to run integration tests. -### See documenation at https://github.com/kubernetes/test-infra/tree/master/prow - -set -ex - -# set cwd correctly -cd "$(dirname "$0")/../" - -# Include requisite scripts -source ./include/util.sh - -TEST_ROOT_DIR=$(git rev-parse --show-toplevel) -BRANCH="master" -SPARK_REPO="https://github.com/apache/spark" -SPARK_REPO_LOCAL_DIR="$TEST_ROOT_DIR/target/spark" - -## Install basic dependencies -## These are for the kubekins-e2e environment in https://github.com/kubernetes/test-infra/tree/master/images/kubekins-e2e -echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -apt-get update && apt-get install -y curl wget git tar uuid-runtime -apt-get install -t jessie-backports -y openjdk-8-jdk - -# Set up config. -MASTER=$(kubectl cluster-info | head -n 1 | grep -oE "https?://[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(:[0-9]+)?") - -# Special GCP project for publishing docker images built by test. -IMAGE_REPO="gcr.io/spark-testing-191023" - -# Cloning the spark distribution -echo "Cloning $SPARK_REPO into $SPARK_REPO_LOCAL_DIR and checking out $BRANCH." -clone_build_spark $SPARK_REPO $SPARK_REPO_LOCAL_DIR $BRANCH - -# Spark distribution -properties=( - -Dspark.kubernetes.test.master=k8s://$MASTER \ - -Dspark.kubernetes.test.imageRepo=$IMAGE_REPO \ - -Dspark.kubernetes.test.sparkTgz="$SPARK_TGZ" \ - -Dspark.kubernetes.test.deployMode=cloud \ - -Dspark.kubernetes.test.namespace=spark \ - -Dspark.kubernetes.test.serviceAccountName=spark-sa -) - -# Run kubectl commands and create appropriate roles -kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user pr-kubekins@kubernetes-jenkins-pull.iam.gserviceaccount.com -kubectl create -f ./dev/spark-rbac.yaml - -# Run tests. -echo "Starting test with ${properties[@]}" -build/mvn integration-test "${properties[@]}" || : - -# Copy out the junit xml files for consumption by k8s test-infra. -ls -1 ./target/surefire-reports/*.xml | cat -n | while read n f; do cp "$f" "/workspace/_artifacts/junit_0$n.xml"; done From 1a531abcf6b8161fd476ce0a8e5e906b5b1a94dc Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 25 May 2018 16:49:11 -0700 Subject: [PATCH 26/39] Remove unused code relating to Kerberos, which doesn't belong in this PR --- .../k8s/integrationtest/KubernetesSuite.scala | 163 ------------------ 1 file changed, 163 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 7353e82569c57..8505f70e13e2b 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -104,112 +104,6 @@ private[spark] class KubernetesSuite extends SparkFunSuite runSparkPiAndVerifyCompletion() } -// test("Run SparkPi with a very long application name.") { -// sparkAppConf.set("spark.app.name", "long" * 40) -// runSparkPiAndVerifyCompletion() -// } -// -// test("Run SparkPi with a master URL without a scheme.") { -// val url = kubernetesTestComponents.kubernetesClient.getMasterUrl -// val k8sMasterUrl = if (url.getPort < 0) { -// s"k8s://${url.getHost}" -// } else { -// s"k8s://${url.getHost}:${url.getPort}" -// } -// sparkAppConf.set("spark.master", k8sMasterUrl) -// runSparkPiAndVerifyCompletion() -// } -// -// test("Run SparkPi with an argument.") { -// runSparkPiAndVerifyCompletion(appArgs = Array("5")) -// } -// -// test("Run SparkPi with custom labels, annotations, and environment variables.") { -// sparkAppConf -// .set("spark.kubernetes.driver.label.label1", "label1-value") -// .set("spark.kubernetes.driver.label.label2", "label2-value") -// .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") -// .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") -// .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") -// .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") -// .set("spark.kubernetes.executor.label.label1", "label1-value") -// .set("spark.kubernetes.executor.label.label2", "label2-value") -// .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") -// .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") -// .set("spark.executorEnv.ENV1", "VALUE1") -// .set("spark.executorEnv.ENV2", "VALUE2") -// -// runSparkPiAndVerifyCompletion( -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkCustomSettings(driverPod) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkCustomSettings(executorPod) -// }) -// } -// -// test("Run SparkPi with a test secret mounted into the driver and executor pods") { -// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") -// createTestSecret(secretName) -// -// sparkAppConf -// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// -// try { -// runSparkPiAndVerifyCompletion( -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkTestSecret(secretName, driverPod) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkTestSecret(secretName, executorPod) -// }) -// } finally { -// deleteTestSecret(secretName) -// } -// } -// -// test("Run PageRank using remote data file") { -// sparkAppConf -// .set("spark.kubernetes.mountDependencies.filesDownloadDir", -// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) -// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) -// runSparkPageRankAndVerifyCompletion( -// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) -// } -// -// test("Run PageRank using remote data file with test secret mounted into the driver and " + -// "executors") { -// val secretName = TEST_SECRET_NAME_PREFIX + UUID.randomUUID().toString.replaceAll("-", "") -// createTestSecret(secretName) -// -// sparkAppConf -// .set("spark.kubernetes.mountDependencies.filesDownloadDir", -// CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) -// .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) -// .set(s"spark.kubernetes.driver.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// .set(s"spark.kubernetes.executor.secrets.$secretName", TEST_SECRET_MOUNT_PATH) -// -// try { -// runSparkPageRankAndVerifyCompletion( -// appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE), -// driverPodChecker = (driverPod: Pod) => { -// doBasicDriverPodCheck(driverPod) -// checkTestSecret(secretName, driverPod, withInitContainer = true) -// }, -// executorPodChecker = (executorPod: Pod) => { -// doBasicExecutorPodCheck(executorPod) -// checkTestSecret(secretName, executorPod, withInitContainer = true) -// }) -// } finally { -// deleteTestSecret(secretName) -// } -// } - private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, driverPodChecker: Pod => Unit = doBasicDriverPodCheck, @@ -324,51 +218,6 @@ private[spark] class KubernetesSuite extends SparkFunSuite .get() == null) } } - - private def createTestSecret(secretName: String): Unit = { - kubernetesTestComponents.kubernetesClient.secrets - .createNew() - .editOrNewMetadata() - .withName(secretName) - .endMetadata() - .addToStringData(TEST_SECRET_KEY, TEST_SECRET_VALUE) - .done() - } - - private def checkTestSecret( - secretName: String, - pod: Pod, - withInitContainer: Boolean = false): Unit = { - val testSecretVolume = pod.getSpec.getVolumes.asScala.filter { volume => - volume.getName == s"$secretName-volume" - } - assert(testSecretVolume.size === 1) - assert(testSecretVolume.head.getSecret.getSecretName === secretName) - - checkTestSecretInContainer(secretName, pod.getSpec.getContainers.get(0)) - - if (withInitContainer) { - checkTestSecretInContainer(secretName, pod.getSpec.getInitContainers.get(0)) - } - } - - private def checkTestSecretInContainer(secretName: String, container: Container): Unit = { - val testSecret = container.getVolumeMounts.asScala.filter { mount => - mount.getName == s"$secretName-volume" - } - assert(testSecret.size === 1) - assert(testSecret.head.getMountPath === TEST_SECRET_MOUNT_PATH) - } - - private def deleteTestSecret(secretName: String): Unit = { - kubernetesTestComponents.kubernetesClient.secrets - .withName(secretName) - .delete() - - Eventually.eventually(TIMEOUT, INTERVAL) { - assert(kubernetesTestComponents.kubernetesClient.secrets.withName(secretName).get() == null) - } - } } private[spark] object KubernetesSuite { @@ -378,17 +227,5 @@ private[spark] object KubernetesSuite { val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" - val TEST_SECRET_NAME_PREFIX = "test-secret-" - val TEST_SECRET_KEY = "test-key" - val TEST_SECRET_VALUE = "test-data" - val TEST_SECRET_MOUNT_PATH = "/etc/secrets" - - val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" - - val REMOTE_PAGE_RANK_DATA_FILE = - "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" - val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = - s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" - case object ShuffleNotReadyException extends Exception } From 1d8a265d13b65dcec8db11a5be09d4a029037d2c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Mon, 28 May 2018 18:53:52 -0700 Subject: [PATCH 27/39] Fix a bug in KubernetesTestComponents - don't an an empty string for zero arguments --- .../k8s/integrationtest/KubernetesTestComponents.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 2df59b6a3373e..09f10937557b2 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -103,14 +103,15 @@ private[spark] object SparkAppLauncher extends Logging { sparkHomeDir: Path): Unit = { val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") - val commandLine = Array(sparkSubmitExecutable.toFile.getAbsolutePath, + val appArgsArray = if (appArguments.appArgs.length > 0) Array(appArguments.appArgs.mkString(" ")) + else Array[String]() + val commandLine = (Array(sparkSubmitExecutable.toFile.getAbsolutePath, "--deploy-mode", "cluster", "--class", appArguments.mainClass, "--master", appConf.get("spark.master") ) ++ appConf.toStringArray :+ - appArguments.mainAppResource :+ - appArguments.appArgs.mkString(" ") - logInfo(s"Launching a spark app with command line: ${commandLine.mkString(" ")}") + appArguments.mainAppResource) ++ + appArgsArray ProcessUtils.executeProcess(commandLine, timeoutSecs) } } From cfb8ee94e11b4871f9b8c7db4774bdb6cb42c40e Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Tue, 29 May 2018 08:41:26 -0700 Subject: [PATCH 28/39] Use Spark's Logging trait --- .../KubernetesTestComponents.scala | 2 ++ .../deploy/k8s/integrationtest/Logging.scala | 35 ------------------- .../k8s/integrationtest/ProcessUtils.scala | 2 ++ .../deploy/k8s/integrationtest/Utils.scala | 2 ++ .../backend/minikube/Minikube.scala | 4 ++- 5 files changed, 9 insertions(+), 36 deletions(-) delete mode 100644 resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 09f10937557b2..3e70c161e32ad 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -25,6 +25,8 @@ import scala.collection.mutable import io.fabric8.kubernetes.client.DefaultKubernetesClient import org.scalatest.concurrent.Eventually +import org.apache.spark.internal.Logging + private[spark] class KubernetesTestComponents(defaultClient: DefaultKubernetesClient) { val namespaceOption = Option(System.getProperty("spark.kubernetes.test.namespace")) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala deleted file mode 100644 index 459c0a4138b86..0000000000000 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Logging.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.deploy.k8s.integrationtest - -import org.apache.log4j.{Logger, LogManager, Priority} - -trait Logging { - - private val log: Logger = LogManager.getLogger(this.getClass) - - protected def logDebug(msg: => String) = if (log.isDebugEnabled) log.debug(msg) - - protected def logInfo(msg: => String) = if (log.isInfoEnabled) log.info(msg) - - protected def logWarning(msg: => String) = if (log.isEnabledFor(Priority.WARN)) log.warn(msg) - - protected def logWarning(msg: => String, throwable: Throwable) = - if (log.isEnabledFor(Priority.WARN)) log.warn(msg, throwable) - - protected def logError(msg: => String) = if (log.isEnabledFor(Priority.ERROR)) log.error(msg) -} diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala index aa6425ddd0353..d8f3a6cec05c3 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/ProcessUtils.scala @@ -21,6 +21,8 @@ import java.util.concurrent.TimeUnit import scala.collection.mutable.ArrayBuffer import scala.io.Source +import org.apache.spark.internal.Logging + object ProcessUtils extends Logging { /** * executeProcess is used to run a command and return the output if it diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala index c300ca46083d4..845ed6167fa27 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -19,6 +19,8 @@ package org.apache.spark.deploy.k8s.integrationtest import java.io.Closeable import java.net.URI +import org.apache.spark.internal.Logging + object Utils extends Logging { def tryWithResource[R <: Closeable, T](createResource: => R)(f: R => T): T = { diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala index 7145d85fb5a1a..c73c7b0412f80 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala @@ -21,7 +21,9 @@ import java.nio.file.Paths import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} -import org.apache.spark.deploy.k8s.integrationtest.{Logging, ProcessUtils} +import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils + +import org.apache.spark.internal.Logging // TODO support windows private[spark] object Minikube extends Logging { From 901edb3ba3a566e5c6737d15e197950abce5131c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Tue, 29 May 2018 08:53:29 -0700 Subject: [PATCH 29/39] Remove trivial duplicate properties from spark root pom.xml --- resource-managers/kubernetes/integration-tests/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index c3d06dadebe20..7cd8791ed72a8 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -27,19 +27,13 @@ spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests - 3.3.9 - 3.5 1.1.1 5.0.2 1.3.0 1.4.0 18.0 - 1.3.9 3.0.0 - 1.2.17 - 2.11.8 - 2.11 3.2.2 2.2.6 1.0 From f68cdba77b323bb829ee69a16b1d3688b45b3129 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Tue, 29 May 2018 09:04:28 -0700 Subject: [PATCH 30/39] Remove unnecessary dependencies (dups from Spark pom.xml) --- .../kubernetes/integration-tests/pom.xml | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 7cd8791ed72a8..4aaacf1847d0b 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -27,17 +27,13 @@ spark-kubernetes-integration-tests_2.11 spark-kubernetes-integration-tests - 1.1.1 - 5.0.2 1.3.0 1.4.0 - 18.0 3.0.0 3.2.2 2.2.6 1.0 - 1.7.24 kubernetes-integration-tests ${project.build.directory}/spark-dist-unpacked N/A @@ -55,7 +51,6 @@ spark-core_${scala.binary.version} ${project.version} - org.apache.spark spark-core_${scala.binary.version} @@ -63,61 +58,11 @@ test-jar test - - - commons-logging - commons-logging - ${commons-logging.version} - - - com.google.code.findbugs - jsr305 - ${jsr305.version} - - - com.google.guava - guava - test - - ${guava.version} - - - com.spotify - docker-client - ${docker-client.version} - test - io.fabric8 kubernetes-client ${kubernetes-client.version} - - log4j - log4j - ${log4j.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - org.scala-lang - scala-library - ${scala.version} - - - org.scalatest - scalatest_${scala.binary.version} - test - - - org.slf4j - slf4j-log4j12 - ${slf4j-log4j12.version} - test - From dd280327ea0cec6d1643007679d189529c4bc1db Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Tue, 29 May 2018 09:10:08 -0700 Subject: [PATCH 31/39] Remove redundant scalatest.version from pom.xml --- resource-managers/kubernetes/integration-tests/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 4aaacf1847d0b..371e44af21559 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -32,7 +32,6 @@ 3.0.0 3.2.2 - 2.2.6 1.0 kubernetes-integration-tests ${project.build.directory}/spark-dist-unpacked From 5a8fd7ff400bcf44e94800905bfa48715eaba88c Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Tue, 29 May 2018 09:14:56 -0700 Subject: [PATCH 32/39] Remove redundant scala maven plugin --- .../kubernetes/integration-tests/pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/pom.xml b/resource-managers/kubernetes/integration-tests/pom.xml index 371e44af21559..520bda89e034d 100644 --- a/resource-managers/kubernetes/integration-tests/pom.xml +++ b/resource-managers/kubernetes/integration-tests/pom.xml @@ -66,19 +66,6 @@ - - net.alchim31.maven - scala-maven-plugin - ${scala-maven-plugin.version} - - - - compile - testCompile - - - - org.codehaus.mojo exec-maven-plugin From 14f2a19cd6d572eb7cd22f80396a8d39d9420166 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 30 May 2018 10:31:04 -0700 Subject: [PATCH 33/39] Fix casing --- .../integrationtest/backend/IntegrationTestBackend.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala index e9a202677654d..284712c6d250e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/IntegrationTestBackend.scala @@ -28,16 +28,16 @@ private[spark] trait IntegrationTestBackend { } private[spark] object IntegrationTestBackendFactory { - val DeployModeConfigKey = "spark.kubernetes.test.deployMode" + val deployModeConfigKey = "spark.kubernetes.test.deployMode" def getTestBackend: IntegrationTestBackend = { - val deployMode = Option(System.getProperty(DeployModeConfigKey)) + val deployMode = Option(System.getProperty(deployModeConfigKey)) .getOrElse("minikube") if (deployMode == "minikube") { MinikubeTestBackend } else { throw new IllegalArgumentException( - "Invalid " + DeployModeConfigKey + ": " + deployMode) + "Invalid " + deployModeConfigKey + ": " + deployMode) } } } From 4102b25537288fd4d6cf8267f07c52a79f49dd72 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Wed, 30 May 2018 17:50:24 -0700 Subject: [PATCH 34/39] Fix scala style issues --- .../deploy/k8s/integrationtest/KubernetesTestComponents.scala | 3 ++- .../deploy/k8s/integrationtest/backend/minikube/Minikube.scala | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index 3e70c161e32ad..ed6edd35cb40a 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -105,7 +105,8 @@ private[spark] object SparkAppLauncher extends Logging { sparkHomeDir: Path): Unit = { val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") - val appArgsArray = if (appArguments.appArgs.length > 0) Array(appArguments.appArgs.mkString(" ")) + val appArgsArray = + if (appArguments.appArgs.length > 0) Array(appArguments.appArgs.mkString(" ")) else Array[String]() val commandLine = (Array(sparkSubmitExecutable.toFile.getAbsolutePath, "--deploy-mode", "cluster", diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala index c73c7b0412f80..6494cbc18f33e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/backend/minikube/Minikube.scala @@ -22,7 +22,6 @@ import java.nio.file.Paths import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient} import org.apache.spark.deploy.k8s.integrationtest.ProcessUtils - import org.apache.spark.internal.Logging // TODO support windows From 67cb6e21cd4726ae398c5f0045b1d758fb6a8238 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Thu, 31 May 2018 08:42:49 -0700 Subject: [PATCH 35/39] Remove whitespace at end of line --- .../deploy/k8s/integrationtest/KubernetesTestComponents.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala index ed6edd35cb40a..48727142dd052 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesTestComponents.scala @@ -105,7 +105,7 @@ private[spark] object SparkAppLauncher extends Logging { sparkHomeDir: Path): Unit = { val sparkSubmitExecutable = sparkHomeDir.resolve(Paths.get("bin", "spark-submit")) logInfo(s"Launching a spark app with arguments $appArguments and conf $appConf") - val appArgsArray = + val appArgsArray = if (appArguments.appArgs.length > 0) Array(appArguments.appArgs.mkString(" ")) else Array[String]() val commandLine = (Array(sparkSubmitExecutable.toFile.getAbsolutePath, From b936953c871226ae8a2ccc7caa6096e9fc38c317 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 1 Jun 2018 12:05:48 -0700 Subject: [PATCH 36/39] Add back in more test cases --- .../k8s/integrationtest/KubernetesSuite.scala | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 8505f70e13e2b..e9322c3d5a388 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -104,6 +104,62 @@ private[spark] class KubernetesSuite extends SparkFunSuite runSparkPiAndVerifyCompletion() } + test("Run SparkPi with a very long application name.") { + sparkAppConf.set("spark.app.name", "long" * 40) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with a master URL without a scheme.") { + val url = kubernetesTestComponents.kubernetesClient.getMasterUrl + val k8sMasterUrl = if (url.getPort < 0) { + s"k8s://${url.getHost}" + } else { + s"k8s://${url.getHost}:${url.getPort}" + } + sparkAppConf.set("spark.master", k8sMasterUrl) + runSparkPiAndVerifyCompletion() + } + + test("Run SparkPi with an argument.") { + runSparkPiAndVerifyCompletion(appArgs = Array("5")) + } + + test("Run SparkPi with custom labels, annotations, and environment variables.") { + sparkAppConf + .set("spark.kubernetes.driver.label.label1", "label1-value") + .set("spark.kubernetes.driver.label.label2", "label2-value") + .set("spark.kubernetes.driver.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.driver.annotation.annotation2", "annotation2-value") + .set("spark.kubernetes.driverEnv.ENV1", "VALUE1") + .set("spark.kubernetes.driverEnv.ENV2", "VALUE2") + .set("spark.kubernetes.executor.label.label1", "label1-value") + .set("spark.kubernetes.executor.label.label2", "label2-value") + .set("spark.kubernetes.executor.annotation.annotation1", "annotation1-value") + .set("spark.kubernetes.executor.annotation.annotation2", "annotation2-value") + .set("spark.executorEnv.ENV1", "VALUE1") + .set("spark.executorEnv.ENV2", "VALUE2") + + runSparkPiAndVerifyCompletion( + driverPodChecker = (driverPod: Pod) => { + doBasicDriverPodCheck(driverPod) + checkCustomSettings(driverPod) + }, + executorPodChecker = (executorPod: Pod) => { + doBasicExecutorPodCheck(executorPod) + checkCustomSettings(executorPod) + }) + } + + // TODO(ssuchter): Enable the below after debugging + //test("Run PageRank using remote data file") { + // sparkAppConf + // .set("spark.kubernetes.mountDependencies.filesDownloadDir", + // CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + // .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + // runSparkPageRankAndVerifyCompletion( + // appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) + //} + private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, driverPodChecker: Pod => Unit = doBasicDriverPodCheck, @@ -227,5 +283,12 @@ private[spark] object KubernetesSuite { val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" + //val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" + + //val REMOTE_PAGE_RANK_DATA_FILE = + // "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" + //val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = + // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" + case object ShuffleNotReadyException extends Exception } From 845cba1db95293d7962fb6029c46e006a5da46a0 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 1 Jun 2018 13:07:36 -0700 Subject: [PATCH 37/39] Fix some scala style issues --- .../k8s/integrationtest/KubernetesSuite.scala | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index e9322c3d5a388..4c73ba4dc7ee7 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -151,14 +151,14 @@ private[spark] class KubernetesSuite extends SparkFunSuite } // TODO(ssuchter): Enable the below after debugging - //test("Run PageRank using remote data file") { - // sparkAppConf - // .set("spark.kubernetes.mountDependencies.filesDownloadDir", - // CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) - // .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) - // runSparkPageRankAndVerifyCompletion( - // appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) - //} + // test("Run PageRank using remote data file") { + // sparkAppConf + // .set("spark.kubernetes.mountDependencies.filesDownloadDir", + // CONTAINER_LOCAL_FILE_DOWNLOAD_PATH) + // .set("spark.files", REMOTE_PAGE_RANK_DATA_FILE) + // runSparkPageRankAndVerifyCompletion( + // appArgs = Array(CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE)) + // } private def runSparkPiAndVerifyCompletion( appResource: String = containerLocalSparkDistroExamplesJar, @@ -283,12 +283,12 @@ private[spark] object KubernetesSuite { val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" - //val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" - - //val REMOTE_PAGE_RANK_DATA_FILE = - // "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" - //val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = - // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" + //val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" + + //val REMOTE_PAGE_RANK_DATA_FILE = + // "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" + //val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = + // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" case object ShuffleNotReadyException extends Exception } From 4c5677a61fd940b818d81469e6640cb45f00ce58 Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 1 Jun 2018 15:07:29 -0700 Subject: [PATCH 38/39] Fix some scala style issues --- .../deploy/k8s/integrationtest/KubernetesSuite.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 4c73ba4dc7ee7..406a20a522487 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -283,12 +283,12 @@ private[spark] object KubernetesSuite { val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi" val SPARK_PAGE_RANK_MAIN_CLASS: String = "org.apache.spark.examples.SparkPageRank" - //val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" + // val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = "/var/spark-data/spark-files" - //val REMOTE_PAGE_RANK_DATA_FILE = - // "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" - //val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = - // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" + // val REMOTE_PAGE_RANK_DATA_FILE = + // "https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt" + // val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = + // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" case object ShuffleNotReadyException extends Exception } From f67d5823b235e04211a9a8d161a3382e14adda8f Mon Sep 17 00:00:00 2001 From: Sean Suchter Date: Fri, 8 Jun 2018 11:50:10 -0700 Subject: [PATCH 39/39] Remove some unused code --- .../k8s/integrationtest/KubernetesSuite.scala | 2 +- .../deploy/k8s/integrationtest/Utils.scala | 58 ------------------- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 406a20a522487..65c513cf241a4 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -290,5 +290,5 @@ private[spark] object KubernetesSuite { // val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE = // s"$CONTAINER_LOCAL_FILE_DOWNLOAD_PATH/pagerank_data.txt" - case object ShuffleNotReadyException extends Exception + // case object ShuffleNotReadyException extends Exception } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala index 845ed6167fa27..663f8b6523ac8 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala @@ -27,62 +27,4 @@ object Utils extends Logging { val resource = createResource try f.apply(resource) finally resource.close() } - - def tryWithSafeFinally[T](block: => T)(finallyBlock: => Unit): T = { - var originalThrowable: Throwable = null - try { - block - } catch { - case t: Throwable => - // Purposefully not using NonFatal, because even fatal exceptions - // we don't want to have our finallyBlock suppress - originalThrowable = t - throw originalThrowable - } finally { - try { - finallyBlock - } catch { - case t: Throwable => - if (originalThrowable != null) { - originalThrowable.addSuppressed(t) - logWarning(s"Suppressing exception in finally: " + t.getMessage, t) - throw originalThrowable - } else { - throw t - } - } - } - } - - def checkAndGetK8sMasterUrl(rawMasterURL: String): String = { - require(rawMasterURL.startsWith("k8s://"), - "Kubernetes master URL must start with k8s://.") - val masterWithoutK8sPrefix = rawMasterURL.substring("k8s://".length) - - // To handle master URLs, e.g., k8s://host:port. - if (!masterWithoutK8sPrefix.contains("://")) { - val resolvedURL = s"https://$masterWithoutK8sPrefix" - logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + - s"URL is $resolvedURL.") - return s"k8s://$resolvedURL" - } - - val masterScheme = new URI(masterWithoutK8sPrefix).getScheme - val resolvedURL = masterScheme.toLowerCase match { - case "https" => - masterWithoutK8sPrefix - case "http" => - logWarning("Kubernetes master URL uses HTTP instead of HTTPS.") - masterWithoutK8sPrefix - case null => - val resolvedURL = s"https://$masterWithoutK8sPrefix" - logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " + - s"URL is $resolvedURL.") - resolvedURL - case _ => - throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme) - } - - s"k8s://$resolvedURL" - } }