Skip to content

[SPARK-26856][PYSPARK][FOLLOWUP] Fix UT failure due to wrong patterns for Kinesis assembly #24268

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[SPARK-26856][PYSPARK][FOLLOWUP] Fix UT failure due to wrong patterns for Kinesis assembly #24268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
690c136
[SPARK-26856][PYSPARK][FOLLOWUP] Fix wrong pattern for Kinesis assembly
dongjoon-hyun Apr 2, 2019
d738df7
Use two parameters.
dongjoon-hyun Apr 2, 2019
0a03c16
Add comments
dongjoon-hyun Apr 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/pyspark/sql/avro/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def _test():
import os
import sys
from pyspark.testing.utils import search_jar
avro_jar = search_jar("external/avro", "spark-avro")
avro_jar = search_jar("external/avro", "spark-avro", "spark-avro")
if avro_jar is None:
print(
"Skipping all Avro Python tests as the optional Avro project was "
Expand Down
3 changes: 2 additions & 1 deletion python/pyspark/testing/streamingutils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
"was not set.")
else:
kinesis_asl_assembly_jar = search_jar("external/kinesis-asl-assembly",
"spark-streaming-kinesis-asl-assembly")
"spark-streaming-kinesis-asl-assembly-",
"spark-streaming-kinesis-asl-assembly_")
if kinesis_asl_assembly_jar is None:
kinesis_requirement_message = (
"Skipping all Kinesis Python tests as the optional Kinesis project was "
Expand Down
8 changes: 5 additions & 3 deletions python/pyspark/testing/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ def close(self):
pass


def search_jar(project_relative_path, jar_name_prefix):
def search_jar(project_relative_path, sbt_jar_name_prefix, mvn_jar_name_prefix):
# Note that 'sbt_jar_name_prefix' and 'mvn_jar_name_prefix' are used since the prefix can
# vary for SBT or Maven specifically. See also SPARK-26856
project_full_path = os.path.join(
os.environ["SPARK_HOME"], project_relative_path)

Expand All @@ -113,9 +115,9 @@ def search_jar(project_relative_path, jar_name_prefix):
# Search jar in the project dir using the jar name_prefix for both sbt build and maven
# build because the artifact jars are in different directories.
sbt_build = glob.glob(os.path.join(
project_full_path, "target/scala-*/%s*.jar" % jar_name_prefix))
project_full_path, "target/scala-*/%s*.jar" % sbt_jar_name_prefix))
maven_build = glob.glob(os.path.join(
project_full_path, "target/%s*.jar" % jar_name_prefix))
project_full_path, "target/%s*.jar" % mvn_jar_name_prefix))
jar_paths = sbt_build + maven_build
jars = [jar for jar in jar_paths if not jar.endswith(ignored_jar_suffixes)]

Expand Down