Improve multi-threading

Author: mdmintz

seleniumbase/core/browser_launcher.py

@@ -1769,17 +1769,27 @@ def _add_chrome_proxy_extension(
     ):
         # Single-threaded
         if zip_it:
-            proxy_helper.create_proxy_ext(
-                proxy_string, proxy_user, proxy_pass, bypass_list
-            )
-            proxy_zip = proxy_helper.PROXY_ZIP_PATH
-            chrome_options.add_extension(proxy_zip)
+            proxy_zip_lock = fasteners.InterProcessLock(PROXY_ZIP_LOCK)
+            with proxy_zip_lock:
+                proxy_helper.create_proxy_ext(
+                    proxy_string, proxy_user, proxy_pass, bypass_list
+                )
+                proxy_zip = proxy_helper.PROXY_ZIP_PATH
+                chrome_options.add_extension(proxy_zip)
         else:
-            proxy_helper.create_proxy_ext(
-                proxy_string, proxy_user, proxy_pass, bypass_list, zip_it=False
-            )
-            proxy_dir_path = proxy_helper.PROXY_DIR_PATH
-            chrome_options = add_chrome_ext_dir(chrome_options, proxy_dir_path)
+            proxy_dir_lock = fasteners.InterProcessLock(PROXY_DIR_LOCK)
+            with proxy_dir_lock:
+                proxy_helper.create_proxy_ext(
+                    proxy_string,
+                    proxy_user,
+                    proxy_pass,
+                    bypass_list,
+                    zip_it=False,
+                )
+                proxy_dir_path = proxy_helper.PROXY_DIR_PATH
+                chrome_options = add_chrome_ext_dir(
+                    chrome_options, proxy_dir_path
+                )
     else:
         # Multi-threaded
         if zip_it:
@@ -1808,7 +1818,7 @@ def _add_chrome_proxy_extension(
                         proxy_user,
                         proxy_pass,
                         bypass_list,
-                        False,
+                        zip_it=False,
                     )
                 chrome_options = add_chrome_ext_dir(
                     chrome_options, proxy_helper.PROXY_DIR_PATH

seleniumbase/core/log_helper.py

@@ -7,6 +7,7 @@
 from seleniumbase import config as sb_config
 from seleniumbase.config import settings
 from seleniumbase.fixtures import constants
+from seleniumbase.fixtures import shared_utils
 
 python3_11_or_newer = False
 if sys.version_info >= (3, 11):
@@ -33,6 +34,8 @@ def log_screenshot(test_logpath, driver, screenshot=None, get=False):
         if screenshot != screenshot_warning:
             with open(screenshot_path, "wb") as file:
                 file.write(screenshot)
+            with suppress(Exception):
+                shared_utils.make_writable(screenshot_path)
         else:
             print("WARNING: %s" % screenshot_warning)
         if get:
@@ -282,13 +285,14 @@ def log_test_failure_data(test, test_logpath, driver, browser, url=None):
         sb_config._report_time = the_time
         sb_config._report_traceback = traceback_message
         sb_config._report_exception = exc_message
-    with suppress(Exception):
-        if not os.path.exists(test_logpath):
+    if not os.path.exists(test_logpath):
+        with suppress(Exception):
             os.makedirs(test_logpath)
     with suppress(Exception):
         log_file = codecs.open(basic_file_path, "w+", encoding="utf-8")
         log_file.writelines("\r\n".join(data_to_save))
         log_file.close()
+        shared_utils.make_writable(basic_file_path)
 
 
 def log_skipped_test_data(test, test_logpath, driver, browser, reason):
@@ -343,6 +347,7 @@ def log_skipped_test_data(test, test_logpath, driver, browser, reason):
         log_file = codecs.open(file_path, "w+", encoding="utf-8")
         log_file.writelines("\r\n".join(data_to_save))
         log_file.close()
+        shared_utils.make_writable(file_path)
 
 
 def log_page_source(test_logpath, driver, source=None):
@@ -365,14 +370,15 @@ def log_page_source(test_logpath, driver, source=None):
                 "unresponsive, or closed prematurely!</h4>"
             )
         )
-    with suppress(Exception):
-        if not os.path.exists(test_logpath):
+    if not os.path.exists(test_logpath):
+        with suppress(Exception):
             os.makedirs(test_logpath)
     html_file_path = os.path.join(test_logpath, html_file_name)
     with suppress(Exception):
         html_file = codecs.open(html_file_path, "w+", encoding="utf-8")
         html_file.write(page_source)
         html_file.close()
+        shared_utils.make_writable(html_file_path)
 
 
 def get_test_id(test):

seleniumbase/core/proxy_helper.py

@@ -2,10 +2,12 @@
 import re
 import warnings
 import zipfile
+from contextlib import suppress
 from seleniumbase.config import proxy_list
 from seleniumbase.config import settings
 from seleniumbase.fixtures import constants
 from seleniumbase.fixtures import page_utils
+from seleniumbase.fixtures import shared_utils
 
 DOWNLOADS_DIR = constants.Files.DOWNLOADS_FOLDER
 PROXY_ZIP_PATH = os.path.join(DOWNLOADS_DIR, "proxy.zip")
@@ -109,45 +111,48 @@ def create_proxy_ext(
         """"minimum_chrome_version":"22.0.0"\n"""
         """}"""
     )
-    import threading
-
-    lock = threading.RLock()  # Support multi-threaded tests. Eg. "pytest -n=4"
-    with lock:
-        abs_path = os.path.abspath(".")
-        downloads_path = os.path.join(abs_path, DOWNLOADS_DIR)
-        if not os.path.exists(downloads_path):
-            os.mkdir(downloads_path)
-        if zip_it:
-            zf = zipfile.ZipFile(PROXY_ZIP_PATH, mode="w")
-            zf.writestr("background.js", background_js)
-            zf.writestr("manifest.json", manifest_json)
-            zf.close()
-        else:
-            proxy_ext_dir = PROXY_DIR_PATH
-            if not os.path.exists(proxy_ext_dir):
-                os.mkdir(proxy_ext_dir)
-            manifest_file = os.path.join(proxy_ext_dir, "manifest.json")
-            with open(manifest_file, mode="w") as f:
-                f.write(manifest_json)
-            proxy_host = proxy_string.split(":")[0]
-            proxy_port = proxy_string.split(":")[1]
-            background_file = os.path.join(proxy_ext_dir, "background.js")
-            with open(background_file, mode="w") as f:
-                f.write(background_js)
+    abs_path = os.path.abspath(".")
+    downloads_path = os.path.join(abs_path, DOWNLOADS_DIR)
+    if not os.path.exists(downloads_path):
+        os.mkdir(downloads_path)
+    if zip_it:
+        zf = zipfile.ZipFile(PROXY_ZIP_PATH, mode="w")
+        zf.writestr("background.js", background_js)
+        zf.writestr("manifest.json", manifest_json)
+        zf.close()
+        with suppress(Exception):
+            shared_utils.make_writable(PROXY_ZIP_PATH)
+    else:
+        proxy_ext_dir = PROXY_DIR_PATH
+        if not os.path.exists(proxy_ext_dir):
+            os.mkdir(proxy_ext_dir)
+        with suppress(Exception):
+            shared_utils.make_writable(proxy_ext_dir)
+        manifest_file = os.path.join(proxy_ext_dir, "manifest.json")
+        with open(manifest_file, mode="w") as f:
+            f.write(manifest_json)
+        with suppress(Exception):
+            shared_utils.make_writable(manifest_json)
+        proxy_host = proxy_string.split(":")[0]
+        proxy_port = proxy_string.split(":")[1]
+        background_file = os.path.join(proxy_ext_dir, "background.js")
+        with open(background_file, mode="w") as f:
+            f.write(background_js)
+        with suppress(Exception):
+            shared_utils.make_writable(background_js)
 
 
 def remove_proxy_zip_if_present():
     """Remove Chromium extension zip file used for proxy server authentication.
     Used in the implementation of https://stackoverflow.com/a/35293284
     for https://stackoverflow.com/questions/12848327/
     """
-    try:
-        if os.path.exists(PROXY_ZIP_PATH):
+    if os.path.exists(PROXY_ZIP_PATH):
+        with suppress(Exception):
             os.remove(PROXY_ZIP_PATH)
-        if os.path.exists(PROXY_ZIP_LOCK):
+    if os.path.exists(PROXY_ZIP_LOCK):
+        with suppress(Exception):
             os.remove(PROXY_ZIP_LOCK)
-    except Exception:
-        pass
 
 
 def validate_proxy_string(proxy_string):

seleniumbase/fixtures/shared_utils.py

@@ -1,6 +1,7 @@
 """Shared utility methods"""
 import colorama
 import os
+import pathlib
 import platform
 import sys
 import time
@@ -128,6 +129,16 @@ def is_chrome_130_or_newer(self, binary_location=None):
     return False
 
 
+def make_dir_files_writable(dir_path):
+    # Make all files in the given directory writable.
+    for file_path in pathlib.Path(dir_path).glob("*"):
+        if file_path.is_file():
+            mode = os.stat(file_path).st_mode
+            mode |= (mode & 0o444) >> 1  # copy R bits to W
+            with suppress(Exception):
+                os.chmod(file_path, mode)
+
+
 def make_writable(file_path):
     # Set permissions to: "If you can read it, you can write it."
     mode = os.stat(file_path).st_mode

seleniumbase/plugins/pytest_plugin.py

@@ -2143,6 +2143,9 @@ def _perform_pytest_unconfigure_(config):
         log_helper.archive_logs_if_set(
             constants.Logs.LATEST + "/", sb_config.archive_logs
         )
+        if os.path.exists("./assets/"):  # Used by pytest-html reports
+            with suppress(Exception):
+                shared_utils.make_dir_files_writable("./assets/")
     log_helper.clear_empty_logs()
     # Dashboard post-processing: Disable time-based refresh and stamp complete
     if not hasattr(sb_config, "dashboard") or not sb_config.dashboard:
@@ -2207,8 +2210,12 @@ def _perform_pytest_unconfigure_(config):
             )
             with open(html_report_path, "w", encoding="utf-8") as f:
                 f.write(the_html_r)  # Finalize the HTML report
+            with suppress(Exception):
+                shared_utils.make_writable(html_report_path)
             with open(html_report_path_copy, "w", encoding="utf-8") as f:
-                f.write(the_html_r)  # Finalize the HTML report
+                f.write(the_html_r)  # Finalize the HTML report copy
+            with suppress(Exception):
+                shared_utils.make_writable(html_report_path_copy)
             assets_style = "./assets/style.css"
             if os.path.exists(assets_style):
                 html_style = None
@@ -2223,6 +2230,8 @@ def _perform_pytest_unconfigure_(config):
                     )
                 with open(assets_style, "w", encoding="utf-8") as f:
                     f.write(html_style)
+                with suppress(Exception):
+                    shared_utils.make_writable(assets_style)
         # Done with "pytest_unconfigure" unless using the Dashboard
         return
     stamp = ""
@@ -2304,6 +2313,8 @@ def _perform_pytest_unconfigure_(config):
                 )
             with open(dashboard_path, "w", encoding="utf-8") as f:
                 f.write(the_html_d)  # Finalize the dashboard
+            with suppress(Exception):
+                shared_utils.make_writable(dashboard_path)
             assets_style = "./assets/style.css"
             if os.path.exists(assets_style):
                 html_style = None
@@ -2318,6 +2329,8 @@ def _perform_pytest_unconfigure_(config):
                     )
                 with open(assets_style, "w", encoding="utf-8") as f:
                     f.write(html_style)
+                with suppress(Exception):
+                    shared_utils.make_writable(assets_style)
             # Part 2: Appending a pytest html report with dashboard data
             html_report_path = None
             if sb_config._html_report_name:
@@ -2398,8 +2411,12 @@ def _perform_pytest_unconfigure_(config):
                 )
                 with open(html_report_path, "w", encoding="utf-8") as f:
                     f.write(the_html_r)  # Finalize the HTML report
+                with suppress(Exception):
+                    shared_utils.make_writable(html_report_path)
                 with open(html_report_path_copy, "w", encoding="utf-8") as f:
-                    f.write(the_html_r)  # Finalize the HTML report
+                    f.write(the_html_r)  # Finalize the HTML report copy
+                with suppress(Exception):
+                    shared_utils.make_writable(html_report_path_copy)
     except KeyboardInterrupt:
         pass
     except Exception: