From 072992fcc5d2f50666beb4e1d88816cd8d82b8f6 Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Fri, 29 May 2020 11:32:17 +0800 Subject: [PATCH 1/2] [FLINK-17857][e2e] Fix Kubernetes e2e tests on Mac OS --- flink-end-to-end-tests/test-scripts/common_docker.sh | 3 ++- .../test-scripts/common_kubernetes.sh | 12 ++++++++++++ .../test-scripts/test_kubernetes_application.sh | 2 +- .../test-scripts/test_kubernetes_embedded_job.sh | 2 +- .../test-scripts/test_kubernetes_session.sh | 2 +- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/flink-end-to-end-tests/test-scripts/common_docker.sh b/flink-end-to-end-tests/test-scripts/common_docker.sh index 4cc3a16418f62..e4dee7d8241ed 100644 --- a/flink-end-to-end-tests/test-scripts/common_docker.sh +++ b/flink-end-to-end-tests/test-scripts/common_docker.sh @@ -35,6 +35,7 @@ function containers_health_check() { function build_image() { local image_name=${1:-flink-job} + local file_server_address=${2:-localhost} echo "Starting fileserver for Flink distribution" pushd ${FLINK_DIR}/.. @@ -47,7 +48,7 @@ function build_image() { echo "Preparing Dockeriles" git clone https://github.com/apache/flink-docker.git --branch dev-master --single-branch cd flink-docker - ./add-custom.sh -u localhost:9999/flink.tgz -n ${image_name} + ./add-custom.sh -u ${file_server_address}:9999/flink.tgz -n ${image_name} echo "Building images" docker build --no-cache --network="host" -t ${image_name} dev/${image_name}-debian diff --git a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh index d6fcacff251fe..b22cc6c7d3f50 100755 --- a/flink-end-to-end-tests/test-scripts/common_kubernetes.sh +++ b/flink-end-to-end-tests/test-scripts/common_kubernetes.sh @@ -94,6 +94,9 @@ function start_kubernetes { echo "$NON_LINUX_ENV_NOTE" exit 1 fi + # Mount Flink dist into minikube virtual machine because we need to mount hostPath as usrlib + minikube mount $FLINK_DIR:$FLINK_DIR & + export minikube_mount_pid=$! else setup_kubernetes_for_linux if ! retry_times ${MINIKUBE_START_RETRIES} ${MINIKUBE_START_BACKOFF} start_kubernetes_if_not_running; then @@ -107,6 +110,7 @@ function start_kubernetes { function stop_kubernetes { if [[ "${OS_TYPE}" != "linux" ]]; then echo "$NON_LINUX_ENV_NOTE" + kill $minikube_mount_pid 2> /dev/null else echo "Stopping minikube ..." stop_command="sudo minikube stop" @@ -175,4 +179,12 @@ appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %-60c %x END } +function get_host_machine_address { + if [[ "${OS_TYPE}" != "linux" ]]; then + echo $(minikube ssh "route -n | grep ^0.0.0.0 | awk '{ print \$2 }' | tr -d '[:space:]'") + else + echo "localhost" + fi +} + on_exit cleanup diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_application.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_application.sh index f480dbf30d7c9..10b7ffd34eb89 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_application.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_application.sh @@ -33,7 +33,7 @@ setConsoleLogging start_kubernetes -build_image ${FLINK_IMAGE_NAME} +build_image ${FLINK_IMAGE_NAME} $(get_host_machine_address) kubectl create clusterrolebinding ${CLUSTER_ROLE_BINDING} --clusterrole=edit --serviceaccount=default:default --namespace=default diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh index d96f4a2e48b0f..0fd76c9ba953e 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_embedded_job.sh @@ -36,7 +36,7 @@ start_kubernetes mkdir -p $OUTPUT_VOLUME -build_image ${FLINK_IMAGE_NAME} +build_image ${FLINK_IMAGE_NAME} $(get_host_machine_address) export USER_LIB=${FLINK_DIR}/examples/batch kubectl create -f ${CONTAINER_SCRIPTS}/job-cluster-service.yaml diff --git a/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh b/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh index a217a13ceae53..00ec5a9dde9bb 100755 --- a/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh +++ b/flink-end-to-end-tests/test-scripts/test_kubernetes_session.sh @@ -47,7 +47,7 @@ setConsoleLogging start_kubernetes -build_image ${FLINK_IMAGE_NAME} +build_image ${FLINK_IMAGE_NAME} $(get_host_machine_address) kubectl create clusterrolebinding ${CLUSTER_ROLE_BINDING} --clusterrole=edit --serviceaccount=default:default --namespace=default From 4c93b68fa64dd913f53d69dad4eab467e78e6507 Mon Sep 17 00:00:00 2001 From: wangyang0918 Date: Tue, 2 Jun 2020 10:09:08 +0800 Subject: [PATCH 2/2] [FLINK-17857][e2e] Fix docker e2e tests on Mac OS --- flink-end-to-end-tests/test-scripts/common_docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flink-end-to-end-tests/test-scripts/common_docker.sh b/flink-end-to-end-tests/test-scripts/common_docker.sh index e4dee7d8241ed..bc771e412c40e 100644 --- a/flink-end-to-end-tests/test-scripts/common_docker.sh +++ b/flink-end-to-end-tests/test-scripts/common_docker.sh @@ -35,7 +35,9 @@ function containers_health_check() { function build_image() { local image_name=${1:-flink-job} - local file_server_address=${2:-localhost} + local default_file_server_address="localhost" + [[ "${OS_TYPE}" != "linux" ]] && default_file_server_address="host.docker.internal" + local file_server_address=${2:-${default_file_server_address}} echo "Starting fileserver for Flink distribution" pushd ${FLINK_DIR}/..