diff --git a/README.md b/README.md index f32d924c0e7..3b74a309af8 100755 --- a/README.md +++ b/README.md @@ -565,7 +565,8 @@ pytest my_first_test.py --pdb --maximize # (Start tests with the browser window maximized.) --screenshot # (Save a screenshot at the end of each test.) --visual-baseline # (Set the visual baseline for Visual/Layout tests.) ---external-pdf # (Set Chrome "plugins.always_open_pdf_externally": True.) +--wire # (Use selenium-wire's webdriver for replacing selenium webdriver.) +--external-pdf # (Set Chromium "plugins.always_open_pdf_externally":True.) --timeout-multiplier=MULTIPLIER # (Multiplies the default timeout values.) --list-fail-page # (After each failing test, list the URL of the failure.) ``` diff --git a/examples/raw_parameter_script.py b/examples/raw_parameter_script.py index f90523aa407..32288b63620 100755 --- a/examples/raw_parameter_script.py +++ b/examples/raw_parameter_script.py @@ -74,6 +74,7 @@ sb._reuse_session = False sb._crumbs = False sb._final_debug = False + sb.use_wire = False sb.visual_baseline = False sb.window_size = None sb.maximize_option = False diff --git a/examples/test_console_logging.py b/examples/test_console_logging.py new file mode 100644 index 00000000000..32a0ce61d4e --- /dev/null +++ b/examples/test_console_logging.py @@ -0,0 +1,13 @@ +from seleniumbase import BaseCase + + +class TestConsoleLogging(BaseCase): + def test_console_logging(self): + self.open("https://seleniumbase.io/demo_page") + self.wait_for_element_visible("h2") + self.start_recording_console_logs() + self.console_log_string("Hello World!") + self.console_log_script('document.querySelector("h2").textContent') + console_logs = [log[0] for log in self.get_recorded_console_logs()] + self.assert_in("Hello World!", console_logs) + self.assert_in("SeleniumBase", console_logs) diff --git a/help_docs/customizing_test_runs.md b/help_docs/customizing_test_runs.md index 54754cb5c1d..6df862f4e59 100755 --- a/help_docs/customizing_test_runs.md +++ b/help_docs/customizing_test_runs.md @@ -179,7 +179,8 @@ pytest my_first_test.py --settings-file=custom_settings.py --maximize # (Start tests with the browser window maximized.) --screenshot # (Save a screenshot at the end of each test.) --visual-baseline # (Set the visual baseline for Visual/Layout tests.) ---external-pdf # (Set Chrome "plugins.always_open_pdf_externally": True.) +--wire # (Use selenium-wire's webdriver for replacing selenium webdriver.) +--external-pdf # (Set Chromium "plugins.always_open_pdf_externally":True.) --timeout-multiplier=MULTIPLIER # (Multiplies the default timeout values.) --list-fail-page # (After each failing test, list the URL of the failure.) ``` diff --git a/help_docs/features_list.md b/help_docs/features_list.md index e48a602459b..81be7d49f81 100755 --- a/help_docs/features_list.md +++ b/help_docs/features_list.md @@ -24,6 +24,7 @@ * Can run tests with a customized browser user agent. (``--agent=USER_AGENT_STRING``) * Can set a Chromium User Data Directory/Profile to load. (``--user-data-dir=DIR``) * Can avoid detection by sites that try to block Selenium. (``--undetected``/``--uc``) +* Can integrate with [selenium-wire](https://github.com/wkeeling/selenium-wire) for inspecting browser requests. (``--wire``) * Can load Chrome Extension ZIP files. (``--extension-zip=ZIP``) * Can load Chrome Extension folders. (``--extension-dir=DIR``) * Powerful [console scripts](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/console_scripts/ReadMe.md). (Type **``seleniumbase``** or **``sbase``** to use.) diff --git a/help_docs/method_summary.md b/help_docs/method_summary.md index 214212182a6..b9e983daf64 100755 --- a/help_docs/method_summary.md +++ b/help_docs/method_summary.md @@ -510,6 +510,16 @@ self.skip(reason="") ############ +self.start_recording_console_logs() + +self.console_log_string(string) + +self.console_log_script(script) + +self.get_recorded_console_logs() + +############ + self.set_local_storage_item(key, value) self.get_local_storage_item(key) @@ -536,6 +546,10 @@ self.get_session_storage_items() ############ +self.set_wire_proxy(string) # Requires "--wire"! + +############ + self.add_css_link(css_link) self.add_js_link(js_link) diff --git a/help_docs/syntax_formats.md b/help_docs/syntax_formats.md index f939cf4104f..6c3013b0a36 100755 --- a/help_docs/syntax_formats.md +++ b/help_docs/syntax_formats.md @@ -73,7 +73,7 @@ class BaseTestCase(BaseCase): # <<< Run custom setUp() code for tests AFTER the super().setUp() >>> def tearDown(self): - self.save_teardown_screenshot() # If test fails, or if "--screenshot" + self.save_teardown_screenshot() # On failure or "--screenshot" if self.has_exception(): # <<< Run custom code if the test failed. >>> pass @@ -255,7 +255,7 @@ class OverrideDriverTest(BaseCase): (From examples/test_override_driver.py) -The above format can let you use [selenium-wire](https://github.com/wkeeling/selenium-wire) to intercept & inspect requests and responses during SeleniumBase tests. Here's how the ``selenium-wire`` integration may look: +The above format lets you customize [selenium-wire](https://github.com/wkeeling/selenium-wire) for intercepting and inspecting requests and responses during SeleniumBase tests. Here's how a ``selenium-wire`` integration may look: ```python from seleniumbase import BaseCase @@ -277,6 +277,8 @@ class WireTestCase(BaseCase): print(request.url) ``` +(NOTE: The ``selenium-wire`` integration is now included with ``seleniumbase``: Add ``--wire`` as a ``pytest`` command-line option to activate. If you need both ``--wire`` with ``--undetected`` together, you'll still need to override ``get_new_driver()``.) +

10. Overriding the driver via "sb" fixture

@@ -297,7 +299,7 @@ def sb(request): super(BaseClass, self).setUp() def tearDown(self): - self.save_teardown_screenshot() + self.save_teardown_screenshot() # On failure or "--screenshot" super(BaseClass, self).tearDown() def base_method(self): @@ -352,7 +354,7 @@ def sb(request): super(BaseClass, self).setUp() def tearDown(self): - self.save_teardown_screenshot() + self.save_teardown_screenshot() # On failure or "--screenshot" super(BaseClass, self).tearDown() def base_method(self): @@ -390,6 +392,8 @@ class TestWire: print(request.url) ``` +(NOTE: The ``selenium-wire`` integration is now included with ``seleniumbase``: Add ``--wire`` as a ``pytest`` command-line option to activate. If you need both ``--wire`` with ``--undetected`` together, you'll still need to override ``get_new_driver()``.) +

11. BaseCase with Chinese translations

diff --git a/requirements.txt b/requirements.txt index 4723c83db05..1f6768a3dce 100755 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ h11==0.14.0;python_version>="3.7" outcome==1.2.0;python_version>="3.7" trio==0.22.0;python_version>="3.7" trio-websocket==0.9.2;python_version>="3.7" -websockets==10.3;python_version>="3.7" +websockets==10.4;python_version>="3.7" pyopenssl==22.1.0;python_version>="3.7" wsproto==1.2.0;python_version>="3.7" selenium==3.141.0;python_version<"3.7" @@ -60,7 +60,8 @@ msedge-selenium-tools==3.141.3;python_version<"3.7" more-itertools==5.0.0;python_version<"3.6" more-itertools==8.14.0;python_version>="3.6" and python_version<"3.7" more-itertools==9.0.0;python_version>="3.7" -cssselect==1.1.0 +cssselect==1.1.0;python_version<"3.7" +cssselect==1.2.0;python_version>="3.7" sortedcontainers==2.4.0 fasteners==0.16;python_version<"3.6" fasteners==0.17.3;python_version>="3.6" and python_version<"3.7" @@ -72,19 +73,20 @@ py==1.8.1;python_version<"3.6" py==1.11.0;python_version>="3.6" pytest==4.6.11;python_version<"3.6" pytest==7.0.1;python_version>="3.6" and python_version<"3.7" -pytest==7.1.3;python_version>="3.7" +pytest==7.2.0;python_version>="3.7" pytest-forked==1.3.0;python_version<"3.6" pytest-forked==1.4.0;python_version>="3.6" pytest-html==1.22.1;python_version<"3.6" pytest-html==2.0.1;python_version>="3.6" pytest-metadata==1.8.0;python_version<"3.6" pytest-metadata==1.11.0;python_version>="3.6" and python_version<"3.7" -pytest-metadata==2.0.2;python_version>="3.7" +pytest-metadata==2.0.3;python_version>="3.7" pytest-ordering==0.6 pytest-rerunfailures==8.0;python_version<"3.6" pytest-rerunfailures==10.2;python_version>="3.6" pytest-xdist==1.34.0;python_version<"3.6" -pytest-xdist==2.5.0;python_version>="3.6" +pytest-xdist==2.5.0;python_version>="3.6" and python_version<"3.7" +pytest-xdist==3.0.2;python_version>="3.7" parameterized==0.8.1 sbvirtualdisplay==1.1.0 behave==1.2.6 @@ -110,6 +112,7 @@ matplotlib-inline==0.1.6;python_version>="3.7" colorama==0.4.6;python_version<"3.6" colorama==0.4.5;python_version>="3.6" and python_version<"3.7" colorama==0.4.6;python_version>="3.7" +exceptiongroup==1.0.0;python_version>="3.7" and python_version<"3.11" importlib-metadata==2.1.3;python_version<"3.6" importlib-metadata==4.2.0;python_version>="3.6" and python_version<"3.8" pycparser==2.21 diff --git a/sbase/__init__.py b/sbase/__init__.py index e69de29bb2d..7a0dd87e230 100755 --- a/sbase/__init__.py +++ b/sbase/__init__.py @@ -0,0 +1,10 @@ +from seleniumbase import BaseCase # noqa +from seleniumbase import decorators # noqa +from seleniumbase import Driver # noqa +from seleniumbase import encryption # noqa +from seleniumbase import get_driver # noqa +from seleniumbase import js_utils # noqa +from seleniumbase import MasterQA # noqa +from seleniumbase import page_actions # noqa +from seleniumbase import page_utils # noqa +from seleniumbase import SB # noqa diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 8020caa472b..bd7dad87601 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.6.6" +__version__ = "4.7.0" diff --git a/seleniumbase/behave/behave_sb.py b/seleniumbase/behave/behave_sb.py index c9e4c4041c6..b47d2612f06 100644 --- a/seleniumbase/behave/behave_sb.py +++ b/seleniumbase/behave/behave_sb.py @@ -89,6 +89,7 @@ -D maximize (Start tests with the browser window maximized.) -D screenshot (Save a screenshot at the end of each test.) -D visual-baseline (Set the visual baseline for Visual/Layout tests.) +-D wire (Use selenium-wire's webdriver for replacing selenium webdriver.) -D external-pdf (Set Chromium "plugins.always_open_pdf_externally": True.) -D timeout-multiplier=MULTIPLIER (Multiplies the default timeout values.) """ @@ -179,6 +180,7 @@ def get_configured_sb(context): sb._crumbs = False sb._disable_beforeunload = False sb.visual_baseline = False + sb.use_wire = False sb.window_size = None sb.maximize_option = False sb.is_context_manager = False @@ -540,6 +542,10 @@ def get_configured_sb(context): if low_key in ["visual-baseline", "visual_baseline"]: sb.visual_baseline = True continue + # Handle: -D wire + if low_key == "wire": + sb.use_wire = True + continue # Handle: -D window-size=Width,Height / window_size=Width,Height if low_key in ["window-size", "window_size"]: window_size = userdata[key] diff --git a/seleniumbase/common/exceptions.py b/seleniumbase/common/exceptions.py index 2d7ee9a69e0..22c515babd0 100755 --- a/seleniumbase/common/exceptions.py +++ b/seleniumbase/common/exceptions.py @@ -4,15 +4,15 @@ OutOfScopeException => Used by BaseCase methods when setUp() is skipped. TextNotVisibleException => Called when expected text fails to appear. TimeLimitExceededException => Called when exceeding "--time-limit=SECONDS". + VisualException => Called when there's a Visual Diff Assertion Failure. """ -from selenium.common.exceptions import WebDriverException class NoSuchFileException(Exception): pass -class NotUsingChromeException(WebDriverException): +class NotUsingChromeException(Exception): pass @@ -20,9 +20,13 @@ class OutOfScopeException(Exception): pass -class TextNotVisibleException(WebDriverException): +class TextNotVisibleException(Exception): pass class TimeLimitExceededException(Exception): pass + + +class VisualException(Exception): + pass diff --git a/seleniumbase/config/proxy_list.py b/seleniumbase/config/proxy_list.py index e46559210d8..0fc5894594e 100755 --- a/seleniumbase/config/proxy_list.py +++ b/seleniumbase/config/proxy_list.py @@ -23,7 +23,7 @@ """ PROXY_LIST = { - "example1": "170.39.193.236:3128", # (Example) - set your own proxy here + "example1": "151.181.91.10:80", # (Example) - set your own proxy here "example2": "socks4://50.197.210.138:32100", # (Example) "proxy1": None, "proxy2": None, diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 6d60eedb24b..6dcb15f69c6 100755 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -16,6 +16,7 @@ from seleniumbase.core import download_helper from seleniumbase.core import proxy_helper from seleniumbase.fixtures import constants +from seleniumbase.fixtures import shared_utils from seleniumbase import drivers # webdriver storage folder for SeleniumBase from seleniumbase import extensions # browser extensions storage folder @@ -372,6 +373,7 @@ def _set_chrome_options( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, @@ -496,7 +498,7 @@ def _set_chrome_options( chrome_options.add_argument("--guest") else: pass - if user_data_dir: + if user_data_dir and not undetectable: abs_path = os.path.abspath(user_data_dir) chrome_options.add_argument("user-data-dir=%s" % abs_path) if extension_zip: @@ -532,7 +534,6 @@ def _set_chrome_options( chrome_options.add_argument( "--disable-blink-features=AutomationControlled" ) - chrome_options.add_experimental_option("useAutomationExtension", False) if headless2: chrome_options.add_argument("--headless=chrome") elif headless: @@ -593,7 +594,8 @@ def _set_chrome_options( chrome_options.add_argument("--ignore-certificate-errors") if not enable_ws: chrome_options.add_argument("--disable-web-security") - chrome_options.add_argument("--no-sandbox") + if "linux" in PLATFORM or not undetectable: + chrome_options.add_argument("--no-sandbox") else: # Opera Chromium only! chrome_options.add_argument("--allow-elevated-browser") @@ -918,6 +920,7 @@ def get_driver( extension_zip=None, extension_dir=None, page_load_strategy=None, + use_wire=False, external_pdf=False, test_id=None, mobile_emulator=False, @@ -937,6 +940,9 @@ def get_driver( headless = True if uc_subprocess and not undetectable: undetectable = True + if undetectable and mobile_emulator: + mobile_emulator = False + user_agent = None proxy_auth = False proxy_user = None proxy_pass = None @@ -1083,6 +1089,7 @@ def get_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, test_id, mobile_emulator, @@ -1130,6 +1137,7 @@ def get_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, mobile_emulator, device_width, @@ -1181,6 +1189,7 @@ def get_remote_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, test_id, mobile_emulator, @@ -1188,6 +1197,21 @@ def get_remote_driver( device_height, device_pixel_ratio, ): + if use_wire and selenium4_or_newer: + driver_fixing_lock = fasteners.InterProcessLock( + constants.MultiBrowser.DRIVER_FIXING_LOCK + ) + with driver_fixing_lock: # Prevent multi-processes mode issues + try: + from seleniumwire import webdriver + except Exception: + shared_utils.pip_install( + "selenium-wire", version=constants.SeleniumWire.VER + ) + from seleniumwire import webdriver + else: + from selenium import webdriver + # Construct the address for connecting to a Selenium Grid if servername.startswith("https://"): protocol = "https" @@ -1280,6 +1304,7 @@ def get_remote_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, @@ -1510,6 +1535,7 @@ def get_remote_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, @@ -1708,6 +1734,7 @@ def get_local_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, mobile_emulator, device_width, @@ -1719,6 +1746,20 @@ def get_local_driver( Can also be used to spin up additional browsers for the same test. """ downloads_path = DOWNLOADS_FOLDER + if use_wire and selenium4_or_newer: + driver_fixing_lock = fasteners.InterProcessLock( + constants.MultiBrowser.DRIVER_FIXING_LOCK + ) + with driver_fixing_lock: # Prevent multi-processes mode issues + try: + from seleniumwire import webdriver + except Exception: + shared_utils.pip_install( + "selenium-wire", version=constants.SeleniumWire.VER + ) + from seleniumwire import webdriver + else: + from selenium import webdriver if browser_name == constants.Browser.FIREFOX: firefox_options = _set_firefox_options( @@ -1987,7 +2028,6 @@ def get_local_driver( edge_options.add_argument( "--disable-blink-features=AutomationControlled" ) - edge_options.add_experimental_option("useAutomationExtension", False) edge_options.add_experimental_option( "excludeSwitches", ["enable-automation", "enable-logging"] ) @@ -2020,7 +2060,7 @@ def get_local_driver( edge_options.add_experimental_option( "mobileEmulation", emulator_settings ) - if user_data_dir: + if user_data_dir and not undetectable: abs_path = os.path.abspath(user_data_dir) edge_options.add_argument("user-data-dir=%s" % abs_path) if extension_zip: @@ -2104,7 +2144,8 @@ def get_local_driver( edge_options.add_argument("--allow-running-insecure-content") if user_agent: edge_options.add_argument("--user-agent=%s" % user_agent) - edge_options.add_argument("--no-sandbox") + if "linux" in PLATFORM or not undetectable: + edge_options.add_argument("--no-sandbox") if remote_debug: # To access the Remote Debugger, go to: http://localhost:9222 # while a Chromium driver is running. @@ -2308,6 +2349,7 @@ def get_local_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, @@ -2372,6 +2414,7 @@ def get_local_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, @@ -2754,6 +2797,7 @@ def get_local_driver( extension_zip, extension_dir, page_load_strategy, + use_wire, external_pdf, servername, mobile_emulator, diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 3b0bfb18fd1..3ba0dd7a132 100755 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -3201,6 +3201,7 @@ def get_new_driver( extension_zip=None, extension_dir=None, page_load_strategy=None, + use_wire=None, external_pdf=None, is_mobile=None, d_width=None, @@ -3251,6 +3252,7 @@ def get_new_driver( extension_zip - A Chrome Extension ZIP file to use (Chrome-only) extension_dir - A Chrome Extension folder to use (Chrome-only) page_load_strategy - the option to change pageLoadStrategy (Chrome) + use_wire - Use selenium-wire webdriver instead of the selenium one external_pdf - "plugins.always_open_pdf_externally": True. (Chrome) is_mobile - the option to use the mobile emulator (Chrome-only) d_width - the device width of the mobile emulator (Chrome-only) @@ -3367,6 +3369,8 @@ def get_new_driver( extension_dir = self.extension_dir if page_load_strategy is None: page_load_strategy = self.page_load_strategy + if use_wire is None: + use_wire = self.use_wire if external_pdf is None: external_pdf = self.external_pdf test_id = self.__get_test_id() @@ -3432,6 +3436,7 @@ def get_new_driver( extension_zip=extension_zip, extension_dir=extension_dir, page_load_strategy=page_load_strategy, + use_wire=use_wire, external_pdf=external_pdf, test_id=test_id, mobile_emulator=is_mobile, @@ -6543,6 +6548,8 @@ def assert_no_js_errors(self, exclude=[]): self.assert_no_js_errors() self.assert_no_js_errors(exclude=["/api.", "/analytics."]) self.assert_no_js_errors(exclude="//api.go,/analytics.go") + self.assert_no_js_errors(exclude=["Uncaught SyntaxError"]) + self.assert_no_js_errors(exclude=["TypeError", "SyntaxE"]) """ self.__check_scope() if ( @@ -6576,10 +6583,6 @@ def assert_no_js_errors(self, exclude=[]): message = message.split( " - Failed to load resource" )[0] - elif message.count(" Uncaught TypeError: ") == 1: - message = message.split( - " Uncaught TypeError: " - )[0] for substring in exclude: substring = str(substring) if ( @@ -6593,10 +6596,15 @@ def assert_no_js_errors(self, exclude=[]): if len(errors) > 0: for n in range(len(errors)): f_t_l_r = " - Failed to load resource" + u_c_s_e = " Uncaught SyntaxError: " u_c_t_e = " Uncaught TypeError: " if f_t_l_r in errors[n]["message"]: url = errors[n]["message"].split(f_t_l_r)[0] errors[n] = {"Error 404 (broken link)": url} + elif u_c_s_e in errors[n]["message"]: + url = errors[n]["message"].split(u_c_s_e)[0] + error = errors[n]["message"].split(u_c_s_e)[1] + errors[n] = {"Uncaught SyntaxError (%s)" % error: url} elif u_c_t_e in errors[n]["message"]: url = errors[n]["message"].split(u_c_t_e)[0] error = errors[n]["message"].split(u_c_t_e)[1] @@ -7634,6 +7642,57 @@ def __assert_shadow_element_visible(self, selector): ############ + # Console Log controls + + def start_recording_console_logs(self): + """ + Starts recording console logs. Logs are saved to: "console.logs". + To get those logs later, call "self.get_recorded_console_logs()". + If navigating to a new page, then the current recorded logs will be + lost, and you'll have to call start_recording_console_logs() again. + # Link1: https://stackoverflow.com/a/19846113/7058266 + # Link2: https://stackoverflow.com/a/74196986/7058266 + """ + self.driver.execute_script( + """ + console.stdlog = console.log.bind(console); + console.logs = []; + console.log = function(){ + console.logs.push(Array.from(arguments)); + console.stdlog.apply(console, arguments); + } + """ + ) + + def console_log_string(self, string): + """ + Log a string to the Web Browser's Console. + Example: + self.console_log_string("Hello World!") + """ + self.driver.execute_script("""console.log(`%s`);""" % string) + + def console_log_script(self, script): + """ + Log output of JavaScript to the Web Browser's Console. + Example: + self.console_log_script('document.querySelector("h2").textContent') + """ + self.driver.execute_script("""console.log(%s);""" % script) + + def get_recorded_console_logs(self): + """ + Returns console logs recorded after "start_recording_console_logs()". + """ + logs = [] + try: + logs = self.driver.execute_script("return console.logs;") + except Exception: + pass + return logs + + ############ + # Application "Local Storage" controls def __is_valid_storage_url(self): @@ -7773,6 +7832,36 @@ def get_session_storage_items(self): ############ + # Methods ONLY for the selenium-wire integration ("--wire") + + def set_wire_proxy(self, string): + """Set a proxy server for selenium-wire mode ("--wire") + NOTE: This method ONLY works while using "--wire" mode! + Examples: + self.set_wire_proxy("SERVER:PORT") + self.set_wire_proxy("socks5://SERVER:PORT") + self.set_wire_proxy("USERNAME:PASSWORD@SERVER:PORT") + """ + if not string: + self.driver.proxy = {} + return + the_http = "http" + the_https = "https" + if string.startswith("socks4://"): + the_http = "socks4" + the_https = "socks4" + elif string.startswith("socks5://"): + the_http = "socks5" + the_https = "socks5" + string = string.split("//")[-1] + self.driver.proxy = { + "http": "%s://%s" % (the_http, string), + "https": "%s://%s" % (the_https, string), + "no_proxy": "localhost,127.0.0.1", + } + + ############ + # Duplicates (Avoids name confusion when migrating from other frameworks.) def open_url(self, url): @@ -11295,7 +11384,9 @@ def __assert_eq(self, *args, **kwargs): elif line.strip().startswith("*"): minified_exception += line + "\n" if minified_exception: - raise Exception(minified_exception) + from seleniumbase.common.exceptions import VisualException + + raise VisualException(minified_exception) def __process_visual_baseline_logs(self): """Save copies of baseline PNGs in "./latest_logs" during failures. @@ -12804,6 +12895,7 @@ def setUp(self, masterqa_mode=False): self.extension_zip = sb_config.extension_zip self.extension_dir = sb_config.extension_dir self.page_load_strategy = sb_config.page_load_strategy + self.use_wire = sb_config.use_wire self.external_pdf = sb_config.external_pdf self._final_debug = sb_config.final_debug self.window_size = sb_config.window_size @@ -13091,6 +13183,7 @@ def setUp(self, masterqa_mode=False): extension_zip=self.extension_zip, extension_dir=self.extension_dir, page_load_strategy=self.page_load_strategy, + use_wire=self.use_wire, external_pdf=self.external_pdf, is_mobile=self.mobile_emulator, d_width=self.__device_width, diff --git a/seleniumbase/fixtures/constants.py b/seleniumbase/fixtures/constants.py index 7516e03d856..72b9cdc6d05 100755 --- a/seleniumbase/fixtures/constants.py +++ b/seleniumbase/fixtures/constants.py @@ -293,6 +293,11 @@ class Tether: ) +class SeleniumWire: + # The version installed if selenium-wire is not installed + VER = "5.1.0" + + class ValidBrowsers: valid_browsers = [ "chrome", diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py index af95fb3cc86..a15035f3fec 100644 --- a/seleniumbase/plugins/driver_manager.py +++ b/seleniumbase/plugins/driver_manager.py @@ -68,14 +68,18 @@ def Driver( extension_zip=None, # Load a Chrome Extension .zip|.crx, comma-separated.) extension_dir=None, # Load a Chrome Extension directory, comma-separated.) page_load_strategy=None, # Set Chrome PLS to "normal", "eager", or "none". + use_wire=None, # Use selenium-wire's webdriver over selenium webdriver. external_pdf=None, # Set Chrome "plugins.always_open_pdf_externally":True. is_mobile=None, # Use the mobile device emulator while running tests. + mobile=None, # Shortcut / Duplicate of "is_mobile". d_width=None, # Set device width d_height=None, # Set device height d_p_r=None, # Set device pixel ratio - uc=None, # Shortcut / Duplicate of "undetectable" to avoid confusion. - undetected=None, # Duplicate of "undetectable" to avoid confusion. - uc_sub=None, # Duplicate of "uc_subprocess" to avoid confusion. + uc=None, # Shortcut / Duplicate of "undetectable". + undetected=None, # Shortcut / Duplicate of "undetectable". + uc_sub=None, # Shortcut / Duplicate of "uc_subprocess". + wire=None, # Shortcut / Duplicate of "use_wire". + pls=None, # Shortcut / Duplicate of "page_load_strategy". ): import sys from seleniumbase.fixtures import constants @@ -202,6 +206,8 @@ def Driver( devtools = True else: devtools = False + if mobile is not None and is_mobile is None: + is_mobile = mobile if is_mobile is None: if "--mobile" in sys_argv: is_mobile = True @@ -277,6 +283,9 @@ def Driver( uc_subprocess = True else: uc_subprocess = False + if undetectable and is_mobile: + is_mobile = False + user_agent = None if use_auto_ext is None: if "--use-auto-ext" in sys_argv: use_auto_ext = True @@ -287,6 +296,8 @@ def Driver( disable_js = True else: disable_js = False + if pls is not None and page_load_strategy is None: + page_load_strategy = pls if page_load_strategy is not None: if page_load_strategy.lower() not in ["normal", "eager", "none"]: raise Exception( @@ -309,6 +320,15 @@ def Driver( do_not_track = True else: do_not_track = False + if use_wire is None and wire is None: + if "--wire" in sys_argv: + use_wire = True + else: + use_wire = False + elif use_wire or wire: + use_wire = True + else: + use_wire = False if external_pdf is None: if "--external-pdf" in sys_argv or "--external_pdf" in sys_argv: external_pdf = True @@ -380,6 +400,7 @@ def Driver( extension_zip=extension_zip, extension_dir=extension_dir, page_load_strategy=page_load_strategy, + use_wire=use_wire, external_pdf=external_pdf, test_id=test_id, mobile_emulator=is_mobile, diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py index 0c64d324dd9..f5c15e4633d 100644 --- a/seleniumbase/plugins/pytest_plugin.py +++ b/seleniumbase/plugins/pytest_plugin.py @@ -68,7 +68,6 @@ def pytest_addoption(parser): --start-page=URL (The starting URL for the web browser when tests begin.) --archive-logs (Archive existing log files instead of deleting them.) --archive-downloads (Archive old downloads instead of deleting them.) - --sjw (Skip JavaScript Waits such as readyState=="complete" or Angular.) --time-limit=SECONDS (Safely fail any test that exceeds the time limit.) --slow (Slow down the automation. Faster than using Demo Mode.) --demo (Slow down and visually see test actions as they occur.) @@ -108,7 +107,8 @@ def pytest_addoption(parser): --maximize (Start tests with the browser window maximized.) --screenshot (Save a screenshot at the end of each test.) --visual-baseline (Set the visual baseline for Visual/Layout tests.) - --external-pdf (Set Chromium "plugins.always_open_pdf_externally": True.) + --wire (Use selenium-wire's webdriver for replacing selenium webdriver.) + --external-pdf (Set Chromium "plugins.always_open_pdf_externally":True.) --timeout-multiplier=MULTIPLIER (Multiplies the default timeout values.) --list-fail-page (After each failing test, list the URL of the failure.) """ @@ -1097,6 +1097,13 @@ def pytest_addoption(parser): When a test calls self.check_window(), it will rebuild its files in the visual_baseline folder.""", ) + parser.addoption( + "--wire", + action="store_true", + dest="use_wire", + default=False, + help="""Use selenium-wire's webdriver for selenium webdriver.""", + ) parser.addoption( "--external_pdf", "--external-pdf", @@ -1266,17 +1273,20 @@ def pytest_addoption(parser): '\n (Your browser choice was: "%s")\n' % browser_list[0] ) raise Exception(message) + undetectable = False + if ( + "--undetected" in sys_argv + or "--undetectable" in sys_argv + or "--uc" in sys_argv + or "--uc-subprocess" in sys_argv + or "--uc_subprocess" in sys_argv + or "--uc-sub" in sys_argv + ): + undetectable = True if ( browser_changes == 1 and browser_text not in ["chrome"] - and ( - "--undetected" in sys_argv - or "--undetectable" in sys_argv - or "--uc" in sys_argv - or "--uc-subprocess" in sys_argv - or "--uc_subprocess" in sys_argv - or "--uc-sub" in sys_argv - ) + and undetectable ): message = ( '\n\n Undetected-Chromedriver Mode ONLY supports Chrome!' @@ -1284,6 +1294,17 @@ def pytest_addoption(parser): '\n (Your browser choice was: "%s")\n' % browser_list[0] ) raise Exception(message) + if undetectable and "--wire" in sys_argv: + raise Exception( + "\n\n SeleniumBase doesn't support mixing --uc with --wire mode!" + "\n If you need both, override get_new_driver() from BaseCase:" + "\n https://seleniumbase.io/help_docs/syntax_formats/#sb_sf_09\n" + ) + if undetectable and "--mobile" in sys_argv: + raise Exception( + "\n\n SeleniumBase doesn't support mixing --uc with --mobile" + '\n UC has: "unrecognized chrome option: mobileEmulation"!\n' + ) def pytest_configure(config): @@ -1415,6 +1436,7 @@ def pytest_configure(config): sb_config.maximize_option = config.getoption("maximize_option") sb_config.save_screenshot = config.getoption("save_screenshot") sb_config.visual_baseline = config.getoption("visual_baseline") + sb_config.use_wire = config.getoption("use_wire") sb_config.external_pdf = config.getoption("external_pdf") sb_config.timeout_multiplier = config.getoption("timeout_multiplier") sb_config.list_fp = config.getoption("fail_page") @@ -1707,7 +1729,10 @@ def pytest_runtest_teardown(item): if ( hasattr(self, "driver") and self.driver - and "--pdb" not in sys_argv + and ( + "--pdb" not in sys_argv + or not python3 + ) ): if not is_windows or self.driver.service.process: self.driver.quit() diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 81d9bfd5a50..9578716d5aa 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -23,8 +23,8 @@ @contextmanager # Usage: -> ``with SB() as sb:`` def SB( test=None, # Test Mode: Output, Logging, Continue on failure unless "rtf". - raise_test_failure=None, # In "test" mode, raise Exception at 1st failure. - rtf=None, # Short form of "raise_test_failure". (Less typing, same thing!) + rtf=None, # Shortcut / Duplicate of "raise_test_failure". + raise_test_failure=None, # If "test" mode, raise Exception on 1st failure. browser=None, # Choose from "chrome", "edge", "firefox", or "safari". headless=None, # The original headless mode for Chromium and Firefox. headless2=None, # Chromium's new headless mode. (Has more features) @@ -59,11 +59,14 @@ def SB( firefox_arg=None, # "ARG=N,ARG2" (Set Firefox args, comma-separated.) firefox_pref=None, # SET (Set Firefox PREFERENCE:VALUE set, ","-separated) user_data_dir=None, # Set the Chrome user data directory to use. - extension_zip=None, # Load a Chrome Extension .zip|.crx, comma-separated.) - extension_dir=None, # Load a Chrome Extension directory, comma-separated.) + extension_zip=None, # Load a Chrome Extension .zip|.crx, comma-separated. + extension_dir=None, # Load a Chrome Extension directory, comma-separated. page_load_strategy=None, # Set Chrome PLS to "normal", "eager", or "none". + skip_js_waits=None, # Skip JS Waits (readyState=="complete" and Angular). + use_wire=None, # Use selenium-wire's webdriver over selenium webdriver. external_pdf=None, # Set Chrome "plugins.always_open_pdf_externally":True. is_mobile=None, # Use the mobile device emulator while running tests. + mobile=None, # Shortcut / Duplicate of "is_mobile". device_metrics=None, # Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio" xvfb=None, # Run tests using the Xvfb virtual display server on Linux OS. start_page=None, # The starting URL for the web browser when tests begin. @@ -82,9 +85,12 @@ def SB( disable_ws=None, # Reverse of "enable_ws". (None and False are different) disable_beforeunload=None, # Disable the "beforeunload" event on Chromium. settings_file=None, # A file for overriding default SeleniumBase settings. - uc=None, # Shortcut / Duplicate of "undetectable" to avoid confusion. - undetected=None, # Duplicate of "undetectable" to avoid confusion. - uc_sub=None, # Duplicate of "uc_subprocess" to avoid confusion. + uc=None, # Shortcut / Duplicate of "undetectable". + undetected=None, # Shortcut / Duplicate of "undetectable". + uc_sub=None, # Shortcut / Duplicate of "uc_subprocess". + wire=None, # Shortcut / Duplicate of "use_wire". + pls=None, # Shortcut / Duplicate of "page_load_strategy". + sjw=None, # Shortcut / Duplicate of "skip_js_waits". save_screenshot=None, # Save a screenshot at the end of each test. timeout_multiplier=None, # Multiplies the default timeout values. js_checking_on=None, # Check for JavaScript errors after page loads. @@ -154,6 +160,7 @@ def SB( raise_test_failure or rtf or "--raise-test-failure" in sys_argv + or "--raise_test_failure" in sys_argv or "--rtf" in sys_argv or "-x" in sys_argv # Carry-over from "pytest" or "--exitfirst" in sys_argv # Carry-over from "pytest" @@ -286,6 +293,8 @@ def SB( devtools = True else: devtools = False + if mobile is not None and is_mobile is None: + is_mobile = mobile if is_mobile is None: if "--mobile" in sys_argv: is_mobile = True @@ -416,6 +425,9 @@ def SB( uc_subprocess = True else: uc_subprocess = False + if undetectable and is_mobile: + is_mobile = False + user_agent = None if use_auto_ext is None: if "--use-auto-ext" in sys_argv: use_auto_ext = True @@ -432,6 +444,8 @@ def SB( _disable_beforeunload = False if disable_beforeunload: _disable_beforeunload = True + if pls is not None and page_load_strategy is None: + page_load_strategy = pls if page_load_strategy is not None: if page_load_strategy.lower() not in ["normal", "eager", "none"]: raise Exception( @@ -444,12 +458,17 @@ def SB( page_load_strategy = "eager" elif "--pls=none" in sys_argv or '--pls="none"' in sys_argv: page_load_strategy = "none" - if ( - "--sjw" in sys_argv - or "--skip_js_waits" in sys_argv - or "--skip-js-waits" in sys_argv - ): - settings.SKIP_JS_WAITS = True + if sjw is not None and skip_js_waits is None: + skip_js_waits = sjw + if skip_js_waits is None: + if ( + "--sjw" in sys_argv + or "--skip_js_waits" in sys_argv + or "--skip-js-waits" in sys_argv + ): + settings.SKIP_JS_WAITS = True + elif skip_js_waits: + settings.SKIP_JS_WAITS = skip_js_waits if save_screenshot is None: if "--screenshot" in sys_argv or "--save-screenshot" in sys_argv: save_screenshot = True @@ -480,6 +499,15 @@ def SB( do_not_track = True else: do_not_track = False + if use_wire is None and wire is None: + if "--wire" in sys_argv: + use_wire = True + else: + use_wire = False + elif use_wire or wire: + use_wire = True + else: + use_wire = False if external_pdf is None: if "--external-pdf" in sys_argv or "--external_pdf" in sys_argv: external_pdf = True @@ -598,6 +626,7 @@ def SB( sb_config.message_duration = message_duration sb_config.block_images = block_images sb_config.do_not_track = do_not_track + sb_config.use_wire = use_wire sb_config.external_pdf = external_pdf sb_config.remote_debug = remote_debug sb_config.settings_file = settings_file @@ -691,6 +720,7 @@ def SB( sb.message_duration = sb_config.message_duration sb.block_images = sb_config.block_images sb.do_not_track = sb_config.do_not_track + sb.use_wire = sb_config.use_wire sb.external_pdf = sb_config.external_pdf sb.remote_debug = sb_config.remote_debug sb.settings_file = sb_config.settings_file diff --git a/seleniumbase/plugins/selenium_plugin.py b/seleniumbase/plugins/selenium_plugin.py index 2aa0e02e57a..f258b3c9eaf 100755 --- a/seleniumbase/plugins/selenium_plugin.py +++ b/seleniumbase/plugins/selenium_plugin.py @@ -82,6 +82,7 @@ class SeleniumBrowser(Plugin): --maximize (Start tests with the browser window maximized.) --screenshot (Save a screenshot at the end of each test.) --visual-baseline (Set the visual baseline for Visual/Layout tests.) + --wire (Use selenium-wire's webdriver for replacing selenium webdriver.) --external-pdf (Set Chromium "plugins.always_open_pdf_externally": True.) --timeout-multiplier=MULTIPLIER (Multiplies the default timeout values.) """ @@ -783,6 +784,13 @@ def options(self, parser, env): When a test calls self.check_window(), it will rebuild its files in the visual_baseline folder.""", ) + parser.add_option( + "--wire", + action="store_true", + dest="use_wire", + default=False, + help="""Use selenium-wire's webdriver for selenium webdriver.""", + ) parser.add_option( "--external_pdf", "--external-pdf", @@ -942,6 +950,7 @@ def beforeTest(self, test): test.test.maximize_option = self.options.maximize_option test.test.save_screenshot_after_test = self.options.save_screenshot test.test.visual_baseline = self.options.visual_baseline + test.test.use_wire = self.options.use_wire test.test.external_pdf = self.options.external_pdf test.test.timeout_multiplier = self.options.timeout_multiplier test.test.dashboard = False @@ -971,6 +980,26 @@ def beforeTest(self, test): ) self.options.headless = True test.test.headless = True + if self.options.use_wire and self.options.undetectable: + print( + "\n" + "SeleniumBase doesn't support mixing --uc with --wire mode.\n" + "If you need both, override get_new_driver() from BaseCase:\n" + "https://seleniumbase.io/help_docs/syntax_formats/#sb_sf_09\n" + "(Only UC Mode without Wire Mode will be used for this run)\n" + ) + self.options.use_wire = False + test.test.use_wire = False + if self.options.mobile_emulator and self.options.undetectable: + print( + "\n" + "SeleniumBase doesn't support mixing --uc with --mobile.\n" + "(Only UC Mode without Mobile will be used for this run)\n" + ) + self.options.mobile_emulator = False + test.test.mobile_emulator = False + self.options.user_agent = None + test.test.user_agent = None # Recorder Mode can still optimize scripts in --headless2 mode. if self.options.recorder_mode and self.options.headless: self.options.headless = False diff --git a/setup.py b/setup.py index eba72ecb382..9cc4510c50a 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ os.system("rm -f dist/*.egg; rm -f dist/*.tar.gz; rm -f dist/*.whl") os.system("rm -rf build/bdist.*; rm -rf build/lib") print("\n*** Installing build: *** (Required for PyPI uploads)\n") - os.system("python -m pip install --upgrade 'build>=0.8.0'") + os.system("python -m pip install --upgrade 'build>=0.9.0'") print("\n*** Installing twine: *** (Required for PyPI uploads)\n") os.system("python -m pip install --upgrade 'twine>=4.0.1'") print("\n*** Installing tqdm: *** (Required for PyPI uploads)\n") @@ -176,7 +176,7 @@ 'outcome==1.2.0;python_version>="3.7"', 'trio==0.22.0;python_version>="3.7"', 'trio-websocket==0.9.2;python_version>="3.7"', - 'websockets==10.3;python_version>="3.7"', + 'websockets==10.4;python_version>="3.7"', 'pyopenssl==22.1.0;python_version>="3.7"', 'wsproto==1.2.0;python_version>="3.7"', 'selenium==3.141.0;python_version<"3.7"', @@ -185,7 +185,8 @@ 'more-itertools==5.0.0;python_version<"3.6"', 'more-itertools==8.14.0;python_version>="3.6" and python_version<"3.7"', # noqa: E501 'more-itertools==9.0.0;python_version>="3.7"', - "cssselect==1.1.0", + 'cssselect==1.1.0;python_version<"3.7"', + 'cssselect==1.2.0;python_version>="3.7"', "sortedcontainers==2.4.0", 'fasteners==0.16;python_version<"3.6"', 'fasteners==0.17.3;python_version>="3.6" and python_version<"3.7"', @@ -197,19 +198,20 @@ 'py==1.11.0;python_version>="3.6"', 'pytest==4.6.11;python_version<"3.6"', 'pytest==7.0.1;python_version>="3.6" and python_version<"3.7"', - 'pytest==7.1.3;python_version>="3.7"', + 'pytest==7.2.0;python_version>="3.7"', 'pytest-forked==1.3.0;python_version<"3.6"', 'pytest-forked==1.4.0;python_version>="3.6"', 'pytest-html==1.22.1;python_version<"3.6"', 'pytest-html==2.0.1;python_version>="3.6"', # Newer ones had issues 'pytest-metadata==1.8.0;python_version<"3.6"', 'pytest-metadata==1.11.0;python_version>="3.6" and python_version<"3.7"', # noqa: E501 - 'pytest-metadata==2.0.2;python_version>="3.7"', + 'pytest-metadata==2.0.3;python_version>="3.7"', "pytest-ordering==0.6", 'pytest-rerunfailures==8.0;python_version<"3.6"', 'pytest-rerunfailures==10.2;python_version>="3.6"', 'pytest-xdist==1.34.0;python_version<"3.6"', - 'pytest-xdist==2.5.0;python_version>="3.6"', + 'pytest-xdist==2.5.0;python_version>="3.6" and python_version<"3.7"', + 'pytest-xdist==3.0.2;python_version>="3.7"', "parameterized==0.8.1", "sbvirtualdisplay==1.1.0", "behave==1.2.6", @@ -235,6 +237,7 @@ 'colorama==0.4.6;python_version<"3.6"', 'colorama==0.4.5;python_version>="3.6" and python_version<"3.7"', 'colorama==0.4.6;python_version>="3.7"', + 'exceptiongroup==1.0.0;python_version>="3.7" and python_version<"3.11"', # noqa: E501 'importlib-metadata==2.1.3;python_version<"3.6"', 'importlib-metadata==4.2.0;python_version>="3.6" and python_version<"3.8"', # noqa: E501 "pycparser==2.21", @@ -259,9 +262,9 @@ 'pytest-cov==2.12.1;python_version<"3.6"', 'pytest-cov==4.0.0;python_version>="3.6"', ], - # pip install -e .[flake] + # pip install -e .[flake8] # Usage: flake8 - "flake": [ + "flake8": [ 'flake8==3.7.9;python_version<"3.6"', 'flake8==5.0.4;python_version>="3.6"', 'mccabe==0.6.1;python_version<"3.6"', @@ -281,7 +284,23 @@ "pillow": [ 'Pillow==6.2.2;python_version<"3.6"', 'Pillow==8.4.0;python_version>="3.6" and python_version<"3.7"', - 'Pillow==9.2.0;python_version>="3.7"', + 'Pillow==9.3.0;python_version>="3.7"', + ], + # pip install -e .[psutil] + "psutil": [ + "psutil==5.9.3", + ], + # pip install -e .[selenium-wire] + "selenium-wire": [ + 'selenium-wire==5.1.0;python_version>="3.7"', + 'Brotli==1.0.9;python_version>="3.7"', + 'blinker==1.5;python_version>="3.7"', + 'h2==4.1.0;python_version>="3.7"', + 'hpack==4.0.0;python_version>="3.7"', + 'hyperframe==6.0.1;python_version>="3.7"', + 'kaitaistruct==0.10;python_version>="3.7"', + 'pyasn1==0.4.8;python_version>="3.7"', + 'zstandard==0.18.0;python_version>="3.7"', ], }, packages=[