From db53aa51bbf9ef10f31ee2ea543f3cbd9bc7f2f8 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 11:00:44 +0900 Subject: [PATCH 1/8] Investigate PIP package failure --- dev/run-pip-tests | 1 + dev/run-tests.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 470f21e69d46..028c0d7924df 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,6 +110,7 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" + ls -al `which spark-submit` spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py diff --git a/dev/run-tests.py b/dev/run-tests.py index 5255a77ec208..c4c14476c627 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -618,13 +618,14 @@ def main(): test_modules = determine_modules_to_test(changed_modules) # license checks - run_apache_rat_checks() + # run_apache_rat_checks() # style checks if not changed_files or any(f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files): - run_scala_style_checks(extra_profiles) + # run_scala_style_checks(extra_profiles) + pass should_run_java_style_checks = False if not changed_files or any(f.endswith(".java") or f.endswith("checkstyle.xml") @@ -636,12 +637,14 @@ def main(): or f.endswith("tox.ini") or f.endswith(".py") for f in changed_files): - run_python_style_checks() + # run_python_style_checks() + pass if not changed_files or any(f.endswith(".R") or f.endswith("lint-r") or f.endswith(".lintr") for f in changed_files): - run_sparkr_style_checks() + # run_sparkr_style_checks() + pass # determine if docs were changed and if we're inside the amplab environment # note - the below commented out until *all* Jenkins workers can get `jekyll` installed @@ -649,7 +652,8 @@ def main(): # build_spark_documentation() if any(m.should_run_build_tests for m in test_modules): - run_build_tests() + # run_build_tests() + pass # spark build build_apache_spark(build_tool, extra_profiles) @@ -663,18 +667,19 @@ def main(): build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks) # run the test suites - run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags) + # run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags) modules_with_python_tests = [m for m in test_modules if m.python_test_goals] if modules_with_python_tests: # We only run PySpark tests with coverage report in one specific job with # Spark master with SBT in Jenkins. is_sbt_master_job = "SPARK_MASTER_SBT_HADOOP_2_7" in os.environ - run_python_tests( - modules_with_python_tests, opts.parallelism, with_coverage=is_sbt_master_job) + # run_python_tests( + # modules_with_python_tests, opts.parallelism, with_coverage=is_sbt_master_job) run_python_packaging_tests() if any(m.should_run_r_tests for m in test_modules): - run_sparkr_tests() + # run_sparkr_tests() + pass def _test(): From 857cdb4195eda2326088f914d12b544d784555b9 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 11:32:08 +0900 Subject: [PATCH 2/8] show permissions --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 028c0d7924df..96edf754f227 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,7 +110,7 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" - ls -al `which spark-submit` + ls -al $(dirname $(dirname $(which ln)))/lib/python3.6/site-packages/pyspark/bin/spark-class spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py From fa9b0e5c3314b6468dfe4910ca22a3fc239f1925 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 11:59:47 +0900 Subject: [PATCH 3/8] typo --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 96edf754f227..d706c9f314d3 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,7 +110,7 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" - ls -al $(dirname $(dirname $(which ln)))/lib/python3.6/site-packages/pyspark/bin/spark-class + ls -al $(dirname $(dirname $(which spark-summit)))/lib/python3.6/site-packages/pyspark/bin/spark-class spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py From 37c32d1ec8826cf102276265e38165c410b5be6b Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 12:28:08 +0900 Subject: [PATCH 4/8] typo :( --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index d706c9f314d3..8eb6898a20d7 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,7 +110,7 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" - ls -al $(dirname $(dirname $(which spark-summit)))/lib/python3.6/site-packages/pyspark/bin/spark-class + ls -al $(dirname $(dirname $(which spark-submit)))/lib/python3.6/site-packages/pyspark/bin/spark-class spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py From f5a60c6ddc0db5ed4544c5dcfe9c106c6a3ead69 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 16:37:40 +0900 Subject: [PATCH 5/8] set upperbound --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 8eb6898a20d7..1c3c3e5bb0d1 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -76,7 +76,7 @@ for python in "${PYTHON_EXECS[@]}"; do VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python rm -rf "$VIRTUALENV_PATH" if [ -n "$USE_CONDA" ]; then - conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools + conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools<46.1.0 source activate "$VIRTUALENV_PATH" else mkdir -p "$VIRTUALENV_PATH" From 01a1cad44df7bb4efefbf0a8753c6a65686cf415 Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 17:04:38 +0900 Subject: [PATCH 6/8] Explicitly set --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 1c3c3e5bb0d1..39cc6fd362b0 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -76,7 +76,7 @@ for python in "${PYTHON_EXECS[@]}"; do VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python rm -rf "$VIRTUALENV_PATH" if [ -n "$USE_CONDA" ]; then - conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools<46.1.0 + conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools=46.0.0 source activate "$VIRTUALENV_PATH" else mkdir -p "$VIRTUALENV_PATH" From 0c83437c06c2fc7b4237f38be2449d5892784f9f Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 17:27:08 +0900 Subject: [PATCH 7/8] Remove comment --- dev/run-pip-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index 39cc6fd362b0..b39bcbc1b58e 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,7 +110,7 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" - ls -al $(dirname $(dirname $(which spark-submit)))/lib/python3.6/site-packages/pyspark/bin/spark-class + # ls -al $(dirname $(dirname $(which spark-submit)))/lib/python3.6/site-packages/pyspark/bin/spark-class spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py From b8926bd63441c49c6c14fe87456036fc74340c4e Mon Sep 17 00:00:00 2001 From: HyukjinKwon Date: Tue, 24 Mar 2020 17:55:04 +0900 Subject: [PATCH 8/8] Revert other changes --- dev/run-pip-tests | 1 - dev/run-tests.py | 23 +++++++++-------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/dev/run-pip-tests b/dev/run-pip-tests index b39bcbc1b58e..dc9582a11f54 100755 --- a/dev/run-pip-tests +++ b/dev/run-pip-tests @@ -110,7 +110,6 @@ for python in "${PYTHON_EXECS[@]}"; do cd / echo "Run basic sanity check on pip installed version with spark-submit" - # ls -al $(dirname $(dirname $(which spark-submit)))/lib/python3.6/site-packages/pyspark/bin/spark-class spark-submit "$FWDIR"/dev/pip-sanity-check.py echo "Run basic sanity check with import based" python3 "$FWDIR"/dev/pip-sanity-check.py diff --git a/dev/run-tests.py b/dev/run-tests.py index c4c14476c627..5255a77ec208 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -618,14 +618,13 @@ def main(): test_modules = determine_modules_to_test(changed_modules) # license checks - # run_apache_rat_checks() + run_apache_rat_checks() # style checks if not changed_files or any(f.endswith(".scala") or f.endswith("scalastyle-config.xml") for f in changed_files): - # run_scala_style_checks(extra_profiles) - pass + run_scala_style_checks(extra_profiles) should_run_java_style_checks = False if not changed_files or any(f.endswith(".java") or f.endswith("checkstyle.xml") @@ -637,14 +636,12 @@ def main(): or f.endswith("tox.ini") or f.endswith(".py") for f in changed_files): - # run_python_style_checks() - pass + run_python_style_checks() if not changed_files or any(f.endswith(".R") or f.endswith("lint-r") or f.endswith(".lintr") for f in changed_files): - # run_sparkr_style_checks() - pass + run_sparkr_style_checks() # determine if docs were changed and if we're inside the amplab environment # note - the below commented out until *all* Jenkins workers can get `jekyll` installed @@ -652,8 +649,7 @@ def main(): # build_spark_documentation() if any(m.should_run_build_tests for m in test_modules): - # run_build_tests() - pass + run_build_tests() # spark build build_apache_spark(build_tool, extra_profiles) @@ -667,19 +663,18 @@ def main(): build_spark_assembly_sbt(extra_profiles, should_run_java_style_checks) # run the test suites - # run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags) + run_scala_tests(build_tool, extra_profiles, test_modules, excluded_tags) modules_with_python_tests = [m for m in test_modules if m.python_test_goals] if modules_with_python_tests: # We only run PySpark tests with coverage report in one specific job with # Spark master with SBT in Jenkins. is_sbt_master_job = "SPARK_MASTER_SBT_HADOOP_2_7" in os.environ - # run_python_tests( - # modules_with_python_tests, opts.parallelism, with_coverage=is_sbt_master_job) + run_python_tests( + modules_with_python_tests, opts.parallelism, with_coverage=is_sbt_master_job) run_python_packaging_tests() if any(m.should_run_r_tests for m in test_modules): - # run_sparkr_tests() - pass + run_sparkr_tests() def _test():