Merge pull request #2094 from seleniumbase/fix-proxy-with-auth-and-bypass-list

Fix proxying with auth, proxy_bypass_list, and Edge notifications

Author: mdmintz

mkdocs_build/requirements.txt

@@ -20,7 +20,7 @@ paginate==0.5.6
 pyquery==2.0.0
 readtime==3.0.0
 mkdocs==1.5.2
-mkdocs-material==9.2.8
+mkdocs-material==9.3.1
 mkdocs-exclude-search==0.6.5
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.1.1

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.18.4"
+__version__ = "4.18.5"

seleniumbase/core/browser_launcher.py

@@ -501,25 +501,29 @@ def _add_chrome_proxy_extension(
     proxy_string,
     proxy_user,
     proxy_pass,
+    proxy_bypass_list=None,
     zip_it=True,
     multi_proxy=False,
 ):
     """Implementation of https://stackoverflow.com/a/35293284 for
     https://stackoverflow.com/questions/12848327/
     (Run Selenium on a proxy server that requires authentication.)"""
     args = " ".join(sys.argv)
+    bypass_list = proxy_bypass_list
     if (
         not ("-n" in sys.argv or " -n=" in args or args == "-c")
         and not multi_proxy
     ):
         # Single-threaded
         if zip_it:
-            proxy_helper.create_proxy_ext(proxy_string, proxy_user, proxy_pass)
+            proxy_helper.create_proxy_ext(
+                proxy_string, proxy_user, proxy_pass, bypass_list
+            )
             proxy_zip = proxy_helper.PROXY_ZIP_PATH
             chrome_options.add_extension(proxy_zip)
         else:
             proxy_helper.create_proxy_ext(
-                proxy_string, proxy_user, proxy_pass, zip_it=False
+                proxy_string, proxy_user, proxy_pass, bypass_list, zip_it=False
             )
             proxy_dir_path = proxy_helper.PROXY_DIR_PATH
             chrome_options = add_chrome_ext_dir(chrome_options, proxy_dir_path)
@@ -532,7 +536,7 @@ def _add_chrome_proxy_extension(
                     _set_proxy_filenames()
                 if not os.path.exists(proxy_helper.PROXY_ZIP_PATH):
                     proxy_helper.create_proxy_ext(
-                        proxy_string, proxy_user, proxy_pass
+                        proxy_string, proxy_user, proxy_pass, bypass_list
                     )
                 proxy_zip = proxy_helper.PROXY_ZIP_PATH
                 chrome_options.add_extension(proxy_zip)
@@ -543,7 +547,11 @@ def _add_chrome_proxy_extension(
                     _set_proxy_filenames()
                 if not os.path.exists(proxy_helper.PROXY_DIR_PATH):
                     proxy_helper.create_proxy_ext(
-                        proxy_string, proxy_user, proxy_pass, False
+                        proxy_string,
+                        proxy_user,
+                        proxy_pass,
+                        bypass_list,
+                        False,
                     )
                 chrome_options = add_chrome_ext_dir(
                     chrome_options, proxy_helper.PROXY_DIR_PATH
@@ -843,6 +851,7 @@ def _set_chrome_options(
                 proxy_string,
                 proxy_user,
                 proxy_pass,
+                proxy_bypass_list,
                 zip_it,
                 multi_proxy,
             )
@@ -861,6 +870,7 @@ def _set_chrome_options(
                 None,
                 proxy_user,
                 proxy_pass,
+                proxy_bypass_list,
                 zip_it,
                 multi_proxy,
             )
@@ -2258,6 +2268,8 @@ def get_local_driver(
             prefs["enable_do_not_track"] = True
         if external_pdf:
             prefs["plugins.always_open_pdf_externally"] = True
+        pdce = "user_experience_metrics.personalization_data_consent_enabled"
+        prefs[pdce] = True  # Remove "Personalize your web experience" prompt
         edge_options.add_experimental_option("prefs", prefs)
         edge_options.add_argument(
             "--disable-blink-features=AutomationControlled"
@@ -2402,6 +2414,7 @@ def get_local_driver(
                     proxy_string,
                     proxy_user,
                     proxy_pass,
+                    proxy_bypass_list,
                     zip_it=True,
                     multi_proxy=multi_proxy,
                 )
@@ -2417,6 +2430,7 @@ def get_local_driver(
                     None,
                     proxy_user,
                     proxy_pass,
+                    proxy_bypass_list,
                     zip_it=True,
                     multi_proxy=multi_proxy,
                 )

seleniumbase/core/proxy_helper.py

@@ -14,14 +14,18 @@
 PROXY_DIR_LOCK = os.path.join(DOWNLOADS_DIR, "proxy_dir.lock")
 
 
-def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True):
+def create_proxy_ext(
+    proxy_string, proxy_user, proxy_pass, bypass_list=None, zip_it=True
+):
     """Implementation of https://stackoverflow.com/a/35293284 for
     https://stackoverflow.com/questions/12848327/
     (Run Selenium on a proxy server that requires authentication.)
     Solution involves creating & adding a Chromium extension at runtime.
     CHROMIUM-ONLY! *** Only Chrome and Edge browsers are supported. ***
     """
     background_js = None
+    if not bypass_list:
+        bypass_list = ""
     if proxy_string:
         proxy_host = proxy_string.split(":")[0]
         proxy_port = proxy_string.split(":")[1]
@@ -31,10 +35,10 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True):
             """    rules: {\n"""
             """      singleProxy: {\n"""
             """        scheme: "http",\n"""
-            """        bypassList: [],\n"""
             """        host: "%s",\n"""
             """        port: parseInt("%s")\n"""
             """      },\n"""
+            """    bypassList: ["%s"]\n"""
             """    }\n"""
             """  };\n"""
             """chrome.proxy.settings.set("""
@@ -52,14 +56,17 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True):
             """        callbackFn,\n"""
             """        {urls: ["<all_urls>"]},\n"""
             """        ['blocking']\n"""
-            """);""" % (proxy_host, proxy_port, proxy_user, proxy_pass)
+            """);""" % (
+                proxy_host, proxy_port, bypass_list, proxy_user, proxy_pass
+            )
         )
     else:
         background_js = (
             """var config = {\n"""
             """    mode: "fixed_servers",\n"""
             """    rules: {\n"""
-            """    }\n"""
+            """    },\n"""
+            """    bypassList: ["%s"]\n"""
             """  };\n"""
             """chrome.proxy.settings.set("""
             """{value: config, scope: "regular"}, function() {"""
@@ -76,7 +83,7 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True):
             """        callbackFn,\n"""
             """        {urls: ["<all_urls>"]},\n"""
             """        ['blocking']\n"""
-            """);""" % (proxy_user, proxy_pass)
+            """);""" % (bypass_list, proxy_user, proxy_pass)
         )
     manifest_json = (
         """{\n"""