From 85ca69de956cd3255eee5c51e830b9aa8f451308 Mon Sep 17 00:00:00 2001 From: rjoshi2 Date: Thu, 8 Feb 2018 21:54:03 -0800 Subject: [PATCH 1/4] [SPARK-23367][Build] Include python document style checking --- dev/lint-python | 28 ++++++++++++++++++++++++++++ dev/skipFiles | 0 dev/tox.ini | 2 ++ 3 files changed, 30 insertions(+) create mode 100644 dev/skipFiles diff --git a/dev/lint-python b/dev/lint-python index f738af9c4976..3a84531cb014 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -21,10 +21,14 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")" # Exclude auto-generated configuration file. PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )" +DOC_PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" | grep -vF 'functions.py')" PYCODESTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-report.txt" +PYDOCSTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-report.txt" PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt" PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt" +PYDOCSTYLEBUILD="pydocstyle" SPHINXBUILD=${SPHINXBUILD:=sphinx-build} + SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt" cd "$SPARK_ROOT_DIR" @@ -82,6 +86,30 @@ else rm "$PYCODESTYLE_REPORT_PATH" fi +# Check python document style, skip check if pydocstyle is not installed. +if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then + pydocstyle --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" + pydocstyle_status="${PIPESTATUS[0]}" + + if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then + lint_status=0 + else + lint_status=1 + fi + + if [ "$lint_status" -ne 0 ]; then + echo "pydocstyle checks failed." + cat "$PYDOCSTYLE_REPORT_PATH" + rm "$PYDOCSTYLE_REPORT_PATH" + exit "$lint_status" + else + echo "pydocstyle checks passed." + rm "$PYDOCSTYLE_REPORT_PATH" + fi +else + echo >&2 "The pydocstyle command was not found. Skipping pydoc checks for now" +fi + # Check that the documentation builds acceptably, skip check if sphinx is not installed. if hash "$SPHINXBUILD" 2> /dev/null; then cd python/docs diff --git a/dev/skipFiles b/dev/skipFiles new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/dev/tox.ini b/dev/tox.ini index 583c1eaaa966..5ad8bb4e08ac 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -17,3 +17,5 @@ ignore=E402,E731,E241,W503,E226,E722,E741,E305 max-line-length=100 exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/* +[pydocstyle] +ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 \ No newline at end of file From ee14cf708603bd904505a110c0ca5d3607d5cdb8 Mon Sep 17 00:00:00 2001 From: rjoshi2 Date: Thu, 8 Feb 2018 22:22:49 -0800 Subject: [PATCH 2/4] [SPARK-23367][Build] Include python document style checking --- dev/lint-python | 37 +++++++++++++++++++++---------------- dev/skipFiles | 0 2 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 dev/skipFiles diff --git a/dev/lint-python b/dev/lint-python index 3a84531cb014..c34923b8af38 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -27,6 +27,7 @@ PYDOCSTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-report.txt" PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt" PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt" PYDOCSTYLEBUILD="pydocstyle" +PYDOCSTYLEVERSION=$(python -c 'import pkg_resources; print pkg_resources.get_distribution("pydocstyle").version') SPHINXBUILD=${SPHINXBUILD:=sphinx-build} SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt" @@ -88,23 +89,27 @@ fi # Check python document style, skip check if pydocstyle is not installed. if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then - pydocstyle --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" - pydocstyle_status="${PIPESTATUS[0]}" - - if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then - lint_status=0 - else - lint_status=1 - fi - - if [ "$lint_status" -ne 0 ]; then - echo "pydocstyle checks failed." - cat "$PYDOCSTYLE_REPORT_PATH" - rm "$PYDOCSTYLE_REPORT_PATH" - exit "$lint_status" + if [ $PYDOCSTYLEVERSION == "2.1.1" ]; then + pydocstyle --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" + pydocstyle_status="${PIPESTATUS[0]}" + + if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then + lint_status=0 + else + lint_status=1 + fi + + if [ "$lint_status" -ne 0 ]; then + echo "pydocstyle checks failed." + cat "$PYDOCSTYLE_REPORT_PATH" + rm "$PYDOCSTYLE_REPORT_PATH" + exit "$lint_status" + else + echo "pydocstyle checks passed." + rm "$PYDOCSTYLE_REPORT_PATH" + fi else - echo "pydocstyle checks passed." - rm "$PYDOCSTYLE_REPORT_PATH" + echo "The pydocstyle version needs to be latest 2.1.1" fi else echo >&2 "The pydocstyle command was not found. Skipping pydoc checks for now" diff --git a/dev/skipFiles b/dev/skipFiles deleted file mode 100644 index e69de29bb2d1..000000000000 From 11ad2c14ac873842299d6bcc2714dcba01b7cc35 Mon Sep 17 00:00:00 2001 From: rjoshi2 Date: Mon, 26 Feb 2018 17:31:17 -0800 Subject: [PATCH 3/4] updated for review comments --- dev/lint-python | 9 +++++---- dev/tox.ini | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/lint-python b/dev/lint-python index c34923b8af38..c8570dd71687 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -21,13 +21,14 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")" # Exclude auto-generated configuration file. PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" )" -DOC_PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" | grep -vF 'functions.py')" +DOC_PATHS_TO_CHECK="$( cd "$SPARK_ROOT_DIR" && find . -name "*.py" | grep -vF 'functions.py' )" PYCODESTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pycodestyle-report.txt" PYDOCSTYLE_REPORT_PATH="$SPARK_ROOT_DIR/dev/pydocstyle-report.txt" PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt" PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt" PYDOCSTYLEBUILD="pydocstyle" -PYDOCSTYLEVERSION=$(python -c 'import pkg_resources; print pkg_resources.get_distribution("pydocstyle").version') +EXPECTED_PYDOCSTYLEVERSION="2.1.1" +PYDOCSTYLEVERSION=$(python -c 'import pkg_resources; print(pkg_resources.get_distribution("pydocstyle").version)' 2> /dev/null) SPHINXBUILD=${SPHINXBUILD:=sphinx-build} SPHINX_REPORT_PATH="$SPARK_ROOT_DIR/dev/sphinx-report.txt" @@ -89,7 +90,7 @@ fi # Check python document style, skip check if pydocstyle is not installed. if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then - if [ $PYDOCSTYLEVERSION == "2.1.1" ]; then + if [ "$PYDOCSTYLEVERSION">="$EXPECTED_PYDOCSTYLEVERSION" ]; then pydocstyle --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" pydocstyle_status="${PIPESTATUS[0]}" @@ -109,7 +110,7 @@ if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then rm "$PYDOCSTYLE_REPORT_PATH" fi else - echo "The pydocstyle version needs to be latest 2.1.1" + echo "The pydocstyle version needs to be latest 2.1.1.Skipping pydoc checks for now" fi else echo >&2 "The pydocstyle command was not found. Skipping pydoc checks for now" diff --git a/dev/tox.ini b/dev/tox.ini index 5ad8bb4e08ac..a9d419068b2e 100644 --- a/dev/tox.ini +++ b/dev/tox.ini @@ -18,4 +18,4 @@ ignore=E402,E731,E241,W503,E226,E722,E741,E305 max-line-length=100 exclude=cloudpickle.py,heapq3.py,shared.py,python/docs/conf.py,work/*/*.py,python/.eggs/* [pydocstyle] -ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 \ No newline at end of file +ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D201,D202,D203,D204,D205,D206,D207,D208,D209,D210,D211,D212,D213,D214,D215,D300,D301,D302,D400,D401,D402,D403,D404,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414 From 5f6b40958229a62ad05fe700a350e435c0c66e7a Mon Sep 17 00:00:00 2001 From: Rekha Joshi Date: Fri, 14 Sep 2018 19:21:26 -0700 Subject: [PATCH 4/4] updated for review comments --- dev/lint-python | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/dev/lint-python b/dev/lint-python index c8570dd71687..67c54662b6a2 100755 --- a/dev/lint-python +++ b/dev/lint-python @@ -73,44 +73,33 @@ python "$PYCODESTYLE_SCRIPT_PATH" --config=dev/tox.ini $PATHS_TO_CHECK >> "$PYCO pycodestyle_status="${PIPESTATUS[0]}" if [ "$compile_status" -eq 0 -a "$pycodestyle_status" -eq 0 ]; then - lint_status=0 + echo "pycodestyle checks passed." + rm "$PYCODESTYLE_REPORT_PATH" else - lint_status=1 -fi - -if [ "$lint_status" -ne 0 ]; then echo "pycodestyle checks failed." cat "$PYCODESTYLE_REPORT_PATH" rm "$PYCODESTYLE_REPORT_PATH" - exit "$lint_status" -else - echo "pycodestyle checks passed." - rm "$PYCODESTYLE_REPORT_PATH" + exit 1 fi # Check python document style, skip check if pydocstyle is not installed. if hash "$PYDOCSTYLEBUILD" 2> /dev/null; then - if [ "$PYDOCSTYLEVERSION">="$EXPECTED_PYDOCSTYLEVERSION" ]; then + if [[ "$PYDOCSTYLEVERSION" == "$EXPECTED_PYDOCSTYLEVERSION" ]]; then pydocstyle --config=dev/tox.ini $DOC_PATHS_TO_CHECK >> "$PYDOCSTYLE_REPORT_PATH" pydocstyle_status="${PIPESTATUS[0]}" if [ "$compile_status" -eq 0 -a "$pydocstyle_status" -eq 0 ]; then - lint_status=0 + echo "pydocstyle checks passed." + rm "$PYDOCSTYLE_REPORT_PATH" else - lint_status=1 - fi - - if [ "$lint_status" -ne 0 ]; then echo "pydocstyle checks failed." cat "$PYDOCSTYLE_REPORT_PATH" rm "$PYDOCSTYLE_REPORT_PATH" - exit "$lint_status" - else - echo "pydocstyle checks passed." - rm "$PYDOCSTYLE_REPORT_PATH" + exit 1 fi + else - echo "The pydocstyle version needs to be latest 2.1.1.Skipping pydoc checks for now" + echo "The pydocstyle version needs to be latest 2.1.1. Skipping pydoc checks for now" fi else echo >&2 "The pydocstyle command was not found. Skipping pydoc checks for now"