[SPARK-37498][PYTHON] Add eventually for test_reuse_worker_of_parallelize_range

Add eventually for test_reuse_worker_of_parallelize_range

Avoid test_reuse_worker_of_parallelize_range becoming flaky when resources are tight or some other reason

No

UT passed.

Closes #35228 from Yikun/SPARK-37498.

Authored-by: Yikun Jiang <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 732477b)
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: Yikun

python/pyspark/tests/test_worker.py

@@ -30,7 +30,7 @@
 from py4j.protocol import Py4JJavaError
 
 from pyspark import SparkConf, SparkContext
-from pyspark.testing.utils import ReusedPySparkTestCase, PySparkTestCase, QuietTest
+from pyspark.testing.utils import ReusedPySparkTestCase, PySparkTestCase, QuietTest, eventually
 
 
 class WorkerTests(ReusedPySparkTestCase):
@@ -181,11 +181,15 @@ def f():
 class WorkerReuseTest(PySparkTestCase):
 
     def test_reuse_worker_of_parallelize_range(self):
-        rdd = self.sc.parallelize(range(20), 8)
-        previous_pids = rdd.map(lambda x: os.getpid()).collect()
-        current_pids = rdd.map(lambda x: os.getpid()).collect()
-        for pid in current_pids:
-            self.assertTrue(pid in previous_pids)
+        def check_reuse_worker_of_parallelize_range():
+            rdd = self.sc.parallelize(range(20), 8)
+            previous_pids = rdd.map(lambda x: os.getpid()).collect()
+            current_pids = rdd.map(lambda x: os.getpid()).collect()
+            for pid in current_pids:
+                self.assertTrue(pid in previous_pids)
+            return True
+
+        eventually(check_reuse_worker_of_parallelize_range, catch_assertions=True)
 
 
 @unittest.skipIf(