From e76d2c78fa3eefdac527d89c7e4a86d4a7073bd7 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 1 Jul 2024 15:12:33 +0900 Subject: [PATCH 1/4] [SPARK-48764][PYTHON] Filtering out IPython frame from user stack --- python/pyspark/errors/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py index 9155bfb54abe8..69b67559a9710 100644 --- a/python/pyspark/errors/utils.py +++ b/python/pyspark/errors/utils.py @@ -175,9 +175,12 @@ def _capture_call_site(spark_session: "SparkSession", depth: int) -> str: # We try import here since IPython is not a required dependency try: - from IPython import get_ipython + import IPython - ipython = get_ipython() + ipython = IPython.get_ipython() + # Filtering out IPython frame + ipy_root = os.path.dirname(IPython.__file__) + selected_frames = [frame for frame in selected_frames if ipy_root not in frame.filename] except ImportError: ipython = None From d4982757bf75287121f91bf2d70a6e29b1d9145a Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 1 Jul 2024 15:25:33 +0900 Subject: [PATCH 2/4] filter ipykernel root --- python/pyspark/errors/utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py index 69b67559a9710..ca8837dacaae2 100644 --- a/python/pyspark/errors/utils.py +++ b/python/pyspark/errors/utils.py @@ -177,10 +177,18 @@ def _capture_call_site(spark_session: "SparkSession", depth: int) -> str: try: import IPython + # ipykernel is required for IPython + import ipykernel + ipython = IPython.get_ipython() # Filtering out IPython frame ipy_root = os.path.dirname(IPython.__file__) - selected_frames = [frame for frame in selected_frames if ipy_root not in frame.filename] + ipykernel_root = os.path.dirname(ipykernel.__file__) + selected_frames = [ + frame + for frame in selected_frames + if (ipy_root not in frame.filename) and (ipykernel_root not in frame.filename) + ] except ImportError: ipython = None From e03fcb1f733db35bed99f822570e7b7516f256d7 Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Mon, 1 Jul 2024 15:25:57 +0900 Subject: [PATCH 3/4] Fix comment --- python/pyspark/errors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py index ca8837dacaae2..9c843de15dc30 100644 --- a/python/pyspark/errors/utils.py +++ b/python/pyspark/errors/utils.py @@ -181,7 +181,7 @@ def _capture_call_site(spark_session: "SparkSession", depth: int) -> str: import ipykernel ipython = IPython.get_ipython() - # Filtering out IPython frame + # Filtering out IPython related frames ipy_root = os.path.dirname(IPython.__file__) ipykernel_root = os.path.dirname(ipykernel.__file__) selected_frames = [ From 8cba7a40792648e951457054360839f32981b9bd Mon Sep 17 00:00:00 2001 From: Haejoon Lee Date: Tue, 2 Jul 2024 08:35:58 +0900 Subject: [PATCH 4/4] fix linter --- python/pyspark/errors/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py index 9c843de15dc30..89721d23c3858 100644 --- a/python/pyspark/errors/utils.py +++ b/python/pyspark/errors/utils.py @@ -178,7 +178,7 @@ def _capture_call_site(spark_session: "SparkSession", depth: int) -> str: import IPython # ipykernel is required for IPython - import ipykernel + import ipykernel # type: ignore[import-not-found] ipython = IPython.get_ipython() # Filtering out IPython related frames