From 21e0218f98419c6636ca9a50e1fc613e1f037aff Mon Sep 17 00:00:00 2001 From: Sahil Prasad Date: Fri, 25 Aug 2017 13:10:19 -0700 Subject: [PATCH 1/3] Modified Python Dockerfiles to allow for the submission of Python applications without the --jars parameter --- .../docker-minimal-bundle/src/main/docker/driver-py/Dockerfile | 2 +- .../src/main/docker/executor-py/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile index 7b1effa911f19..c80dea7bfb1eb 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \ +CMD SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \ diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile index a8bb5b362ab52..dd0c3b38dc68d 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/*" && \ +CMD SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \ From 78cf5aeea3d2b592aa74884dc820f06859efd5a3 Mon Sep 17 00:00:00 2001 From: Sahil Prasad Date: Thu, 28 Sep 2017 10:58:32 -0400 Subject: [PATCH 2/3] Adding quotes to JARS wildcard in place of SPARK_CLASSPATH addition --- .../docker-minimal-bundle/src/main/docker/driver-py/Dockerfile | 2 +- .../src/main/docker/executor-py/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile index c80dea7bfb1eb..2898003fc0d23 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" && \ +CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/\"*\"" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \ diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile index dd0c3b38dc68d..c312028e52097 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="$SPARK_CLASSPATH:${SPARK_HOME}/jars/*" && \ +CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/\"*\"" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \ From 52427df23ce62e794e0297b56b7a4f8b8816fa44 Mon Sep 17 00:00:00 2001 From: Sahil Prasad Date: Sun, 1 Oct 2017 23:45:14 -0400 Subject: [PATCH 3/3] Fixed failing integration tests --- .../docker-minimal-bundle/src/main/docker/driver-py/Dockerfile | 2 +- .../src/main/docker/executor-py/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile index 2898003fc0d23..9e5f26720ae5c 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/driver-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/\"*\"" && \ +CMD SPARK_CLASSPATH=":${SPARK_HOME}/jars/*" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \ diff --git a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile index c312028e52097..48df792e3dab9 100644 --- a/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile +++ b/resource-managers/kubernetes/docker-minimal-bundle/src/main/docker/executor-py/Dockerfile @@ -38,7 +38,7 @@ ENV PYSPARK_PYTHON python ENV PYSPARK_DRIVER_PYTHON python ENV PYTHONPATH ${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${PYTHONPATH} -CMD SPARK_CLASSPATH="${SPARK_HOME}/jars/\"*\"" && \ +CMD SPARK_CLASSPATH=":${SPARK_HOME}/jars/*" && \ env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\(.*\)/\1/g' > /tmp/java_opts.txt && \ readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && \ if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH="$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH"; fi && \