[SPARK-8706] [PYSPARK] [PROJECT INFRA] Add pylint checks to PySpark

This adds Pylint checks to PySpark.

For now this lazy installs using easy_install to /dev/pylint (similar to the pep8 script).
We still need to figure out what rules to be allowed.

Author: MechCoder <[email protected]>

Closes #7241 from MechCoder/pylint and squashes the following commits:

2fc7291 [MechCoder] Remove pylint test fail
6d883a2 [MechCoder] Silence warnings and make pylint tests fail to check if it works in jenkins
f3a5e17 [MechCoder] undefined-variable
ca8b749 [MechCoder] Minor changes
71629f8 [MechCoder] remove trailing whitespace
8498ff9 [MechCoder] Remove blacklisted arguments and pointless statements check
1dbd094 [MechCoder] Disable all checks for now
8b8aa8a [MechCoder] Add pylint configuration file
7871bb1 [MechCoder] [SPARK-8706] [PySpark] [Project infra] Add pylint checks to PySpark

Author: MechCoder

dev/lint-python

@@ -21,12 +21,14 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 SPARK_ROOT_DIR="$(dirname "$SCRIPT_DIR")"
 PATHS_TO_CHECK="./python/pyspark/ ./ec2/spark_ec2.py ./examples/src/main/python/ ./dev/sparktestsupport"
 PATHS_TO_CHECK="$PATHS_TO_CHECK ./dev/run-tests.py ./python/run-tests.py"
-PYTHON_LINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/python-lint-report.txt"
+PEP8_REPORT_PATH="$SPARK_ROOT_DIR/dev/pep8-report.txt"
+PYLINT_REPORT_PATH="$SPARK_ROOT_DIR/dev/pylint-report.txt"
+PYLINT_INSTALL_INFO="$SPARK_ROOT_DIR/dev/pylint-info.txt"
 
 cd "$SPARK_ROOT_DIR"
 
 # compileall: https://docs.python.org/2/library/compileall.html
-python -B -m compileall -q -l $PATHS_TO_CHECK > "$PYTHON_LINT_REPORT_PATH"
+python -B -m compileall -q -l $PATHS_TO_CHECK > "$PEP8_REPORT_PATH"
 compile_status="${PIPESTATUS[0]}"
 
 # Get pep8 at runtime so that we don't rely on it being installed on the build server.
@@ -47,11 +49,36 @@ if [ ! -e "$PEP8_SCRIPT_PATH" ]; then
     fi
 fi
 
+# Easy install pylint in /dev/pylint. To easy_install into a directory, the PYTHONPATH should
+# be set to the directory.
+# dev/pylint should be appended to the PATH variable as well.
+# Jenkins by default installs the pylint3 version, so for now this just checks the code quality
+# of python3.
+export "PYTHONPATH=$SPARK_ROOT_DIR/dev/pylint"
+export "PYLINT_HOME=$PYTHONPATH"
+export "PATH=$PYTHONPATH:$PATH"
+
+if [ ! -d "$PYLINT_HOME" ]; then
+    mkdir "$PYLINT_HOME"
+    # Redirect the annoying pylint installation output.
+    easy_install -d "$PYLINT_HOME" pylint==1.4.4 &>> "$PYLINT_INSTALL_INFO"
+    easy_install_status="$?"
+
+    if [ "$easy_install_status" -ne 0 ]; then
+        echo "Unable to install pylint locally in \"$PYTHONPATH\"."
+        cat "$PYLINT_INSTALL_INFO"
+        exit "$easy_install_status"
+    fi
+
+    rm "$PYLINT_INSTALL_INFO"
+
+fi
+
 # There is no need to write this output to a file
 #+ first, but we do so so that the check status can
 #+ be output before the report, like with the
 #+ scalastyle and RAT checks.
-python "$PEP8_SCRIPT_PATH" --ignore=E402,E731,E241,W503,E226 $PATHS_TO_CHECK >> "$PYTHON_LINT_REPORT_PATH"
+python "$PEP8_SCRIPT_PATH" --ignore=E402,E731,E241,W503,E226 $PATHS_TO_CHECK >> "$PEP8_REPORT_PATH"
 pep8_status="${PIPESTATUS[0]}"
 
 if [ "$compile_status" -eq 0 -a "$pep8_status" -eq 0 ]; then
@@ -61,13 +88,27 @@ else
 fi
 
 if [ "$lint_status" -ne 0 ]; then
-    echo "Python lint checks failed."
-    cat "$PYTHON_LINT_REPORT_PATH"
+    echo "PEP8 checks failed."
+    cat "$PEP8_REPORT_PATH"
+else
+    echo "PEP8 checks passed."
+fi
+
+rm "$PEP8_REPORT_PATH"
+
+for to_be_checked in "$PATHS_TO_CHECK"
+do
+    pylint --rcfile="$SPARK_ROOT_DIR/pylintrc" $to_be_checked >> "$PYLINT_REPORT_PATH"
+done
+
+if [ "${PIPESTATUS[0]}" -ne 0 ]; then
+    lint_status=1
+    echo "Pylint checks failed."
+    cat "$PYLINT_REPORT_PATH"
 else
-    echo "Python lint checks passed."
+    echo "Pylint checks passed."
 fi
 
-# rm "$PEP8_SCRIPT_PATH"
-rm "$PYTHON_LINT_REPORT_PATH"
+rm "$PYLINT_REPORT_PATH"
 
 exit "$lint_status"