From 1067d88ee8ea52f3a905cd854176399d2de134b6 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 21 Mar 2024 17:30:14 -0400 Subject: [PATCH 1/4] Make UC Mode great again --- seleniumbase/core/browser_launcher.py | 33 ++++++++++++++--------- seleniumbase/fixtures/base_case.py | 22 +-------------- seleniumbase/fixtures/constants.py | 2 +- seleniumbase/fixtures/js_utils.py | 39 +++++++++++++++++++++++++++ seleniumbase/undetected/webelement.py | 13 ++++++--- 5 files changed, 72 insertions(+), 37 deletions(-) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 14687ccad12..b666631a332 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -463,11 +463,16 @@ def uc_click( except Exception: pass element = driver.wait_for_selector(selector, by=by, timeout=timeout) - if not element.tag_name == "span": # Element must be "visible" + tag_name = element.tag_name + if not tag_name == "span": # Element must be "visible" element = driver.wait_for_element(selector, by=by, timeout=timeout) try: element.uc_click( - driver, selector, by=by, reconnect_time=reconnect_time + driver, + selector, + by=by, + reconnect_time=reconnect_time, + tag_name=tag_name, ) except ElementClickInterceptedException: time.sleep(0.16) @@ -812,11 +817,12 @@ def _set_chrome_options( chrome_options = webdriver.edge.options.Options() prefs = {} prefs["download.default_directory"] = downloads_path - prefs["local_discovery.notifications_enabled"] = False - prefs["credentials_enable_service"] = False - prefs["download.prompt_for_download"] = False prefs["download.directory_upgrade"] = True + prefs["download.prompt_for_download"] = False + prefs["credentials_enable_service"] = False + prefs["local_discovery.notifications_enabled"] = False prefs["safebrowsing.enabled"] = False + prefs["safebrowsing.disable_download_protection"] = True prefs["omnibox-max-zero-suggest-matches"] = 0 prefs["omnibox-use-existing-autocomplete-client"] = 0 prefs["omnibox-trending-zero-prefix-suggestions-on-ntp"] = 0 @@ -827,9 +833,8 @@ def _set_chrome_options( prefs["omnibox-zero-suggest-prefetching-on-srp"] = 0 prefs["omnibox-zero-suggest-prefetching-on-web"] = 0 prefs["omnibox-zero-suggest-in-memory-caching"] = 0 - prefs["default_content_setting_values.notifications"] = 0 prefs["content_settings.exceptions.automatic_downloads.*.setting"] = 1 - prefs["safebrowsing.disable_download_protection"] = True + prefs["default_content_setting_values.notifications"] = 0 prefs["default_content_settings.popups"] = 0 prefs["managed_default_content_settings.popups"] = 0 prefs["profile.password_manager_enabled"] = False @@ -1142,6 +1147,7 @@ def _set_chrome_options( included_disabled_features.append("PrivacySandboxSettings4") included_disabled_features.append("DownloadBubble") included_disabled_features.append("DownloadBubbleV2") + included_disabled_features.append("InsecureDownloadWarnings") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -1153,6 +1159,7 @@ def _set_chrome_options( included_disabled_features.append("OptimizationTargetPrediction") included_disabled_features.append("DownloadBubble") included_disabled_features.append("DownloadBubbleV2") + included_disabled_features.append("InsecureDownloadWarnings") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -2367,10 +2374,11 @@ def get_local_driver( elif browser_name == constants.Browser.EDGE: prefs = { "download.default_directory": downloads_path, - "local_discovery.notifications_enabled": False, - "credentials_enable_service": False, - "download.prompt_for_download": False, "download.directory_upgrade": True, + "download.prompt_for_download": False, + "credentials_enable_service": False, + "local_discovery.notifications_enabled": False, + "safebrowsing.disable_download_protection": True, "safebrowsing.enabled": False, "omnibox-max-zero-suggest-matches": 0, "omnibox-use-existing-autocomplete-client": 0, @@ -2382,11 +2390,10 @@ def get_local_driver( "omnibox-zero-suggest-prefetching-on-srp": 0, "omnibox-zero-suggest-prefetching-on-web": 0, "omnibox-zero-suggest-in-memory-caching": 0, - "safebrowsing.disable_download_protection": True, + "content_settings.exceptions.automatic_downloads.*.setting": 1, "default_content_setting_values.notifications": 0, "default_content_settings.popups": 0, "managed_default_content_settings.popups": 0, - "content_settings.exceptions.automatic_downloads.*.setting": 1, "profile.password_manager_enabled": False, "profile.default_content_setting_values.notifications": 2, "profile.default_content_settings.popups": 0, @@ -2772,6 +2779,7 @@ def get_local_driver( included_disabled_features.append("Translate") included_disabled_features.append("OptimizationTargetPrediction") included_disabled_features.append("PrivacySandboxSettings4") + included_disabled_features.append("InsecureDownloadWarnings") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -2781,6 +2789,7 @@ def get_local_driver( included_disabled_features.append("OptimizationHintsFetching") included_disabled_features.append("Translate") included_disabled_features.append("OptimizationTargetPrediction") + included_disabled_features.append("InsecureDownloadWarnings") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index c84ac721345..20940f0f4fb 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -7876,27 +7876,7 @@ def convert_to_css_selector(self, selector, by): jQuery commands require a CSS_SELECTOR for finding elements. This method should only be used for jQuery/JavaScript actions. Pure JavaScript doesn't support using a:contains("LINK_TEXT").""" - if by == By.CSS_SELECTOR: - return selector - elif by == By.ID: - return "#%s" % selector - elif by == By.CLASS_NAME: - return ".%s" % selector - elif by == By.NAME: - return '[name="%s"]' % selector - elif by == By.TAG_NAME: - return selector - elif by == By.XPATH: - return self.convert_xpath_to_css(selector) - elif by == By.LINK_TEXT: - return 'a:contains("%s")' % selector - elif by == By.PARTIAL_LINK_TEXT: - return 'a:contains("%s")' % selector - else: - raise Exception( - "Exception: Could not convert {%s}(by=%s) to CSS_SELECTOR!" - % (selector, by) - ) + return js_utils.convert_to_css_selector(selector, by) def set_value( self, selector, text, by="css selector", timeout=None, scroll=True diff --git a/seleniumbase/fixtures/constants.py b/seleniumbase/fixtures/constants.py index 3cc130b059d..fd920bbfbfa 100644 --- a/seleniumbase/fixtures/constants.py +++ b/seleniumbase/fixtures/constants.py @@ -367,7 +367,7 @@ class Mobile: class UC: - RECONNECT_TIME = 2.35 # Seconds + RECONNECT_TIME = 2.4 # Seconds class ValidBrowsers: diff --git a/seleniumbase/fixtures/js_utils.py b/seleniumbase/fixtures/js_utils.py index 4f6261d3abb..5ad4aaf8438 100644 --- a/seleniumbase/fixtures/js_utils.py +++ b/seleniumbase/fixtures/js_utils.py @@ -4,10 +4,12 @@ import time from selenium.common.exceptions import NoSuchElementException from selenium.common.exceptions import WebDriverException +from selenium.webdriver.common.by import By from seleniumbase import config as sb_config from seleniumbase.config import settings from seleniumbase.fixtures import constants from seleniumbase.fixtures import css_to_xpath +from seleniumbase.fixtures import xpath_to_css def wait_for_ready_state_complete(driver, timeout=settings.LARGE_TIMEOUT): @@ -93,6 +95,37 @@ def wait_for_angularjs(driver, timeout=settings.LARGE_TIMEOUT, **kwargs): time.sleep(0.05) +def convert_to_css_selector(selector, by=By.CSS_SELECTOR): + if by == By.CSS_SELECTOR: + return selector + elif by == By.ID: + return "#%s" % selector + elif by == By.CLASS_NAME: + return ".%s" % selector + elif by == By.NAME: + return '[name="%s"]' % selector + elif by == By.TAG_NAME: + return selector + elif ( + by == By.XPATH + or ( + selector.startswith("/") + or selector.startswith("./") + or selector.startswith("(") + ) + ): + return xpath_to_css.convert_xpath_to_css(selector) + elif by == By.LINK_TEXT: + return 'a:contains("%s")' % selector + elif by == By.PARTIAL_LINK_TEXT: + return 'a:contains("%s")' % selector + else: + raise Exception( + "Exception: Could not convert {%s}(by=%s) to CSS_SELECTOR!" + % (selector, by) + ) + + def is_html_inspector_activated(driver): try: driver.execute_script("HTMLInspector;") # Fails if not defined @@ -325,6 +358,12 @@ def swap_selector_and_by_if_reversed(selector, by): return (selector, by) +def call_me_later(driver, script, ms): + """Call script after ms passed.""" + call = "function() {%s}" % script + driver.execute_script("window.setTimeout(%s, %s);" % (call, ms)) + + def highlight(driver, selector, by="css selector", loops=4): """For driver.highlight() / driver.page.highlight()""" swap_selector_and_by_if_reversed(selector, by) diff --git a/seleniumbase/undetected/webelement.py b/seleniumbase/undetected/webelement.py index c85a44fbd52..51af3326248 100644 --- a/seleniumbase/undetected/webelement.py +++ b/seleniumbase/undetected/webelement.py @@ -1,4 +1,5 @@ import selenium.webdriver.remote.webelement +from seleniumbase.fixtures import js_utils class WebElement(selenium.webdriver.remote.webelement.WebElement): @@ -8,18 +9,24 @@ def uc_click( selector=None, by=None, reconnect_time=None, + tag_name=None, ): if driver and selector and by: - driver.js_click(selector, by=by, timeout=1) + if tag_name == "span" and ":contains" not in selector: + selector = js_utils.convert_to_css_selector(selector, by) + script = 'document.querySelector("%s").click();' % selector + js_utils.call_me_later(driver, script, 111) + else: + driver.js_click(selector, by=by, timeout=1) else: super().click() if not reconnect_time: - self._parent.reconnect(0.45) + self._parent.reconnect(0.5) else: self._parent.reconnect(reconnect_time) def uc_reconnect(self, reconnect_time=None): if not reconnect_time: - self._parent.reconnect(0.1) + self._parent.reconnect(0.2) else: self._parent.reconnect(reconnect_time) From 4820d73b066500e543dd78eebde0869f5e7ffac2 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 21 Mar 2024 17:30:52 -0400 Subject: [PATCH 2/4] Update example tests --- examples/raw_cdp_logging.py | 2 +- examples/raw_form_turnstile.py | 24 ++++-------------------- examples/raw_nopecha.py | 8 ++++---- examples/raw_turnstile.py | 6 +++--- examples/test_download_files.py | 1 + examples/uc_cdp_events.py | 6 +++--- 6 files changed, 16 insertions(+), 31 deletions(-) diff --git a/examples/raw_cdp_logging.py b/examples/raw_cdp_logging.py index bfbff0bad42..5f7965ed73a 100644 --- a/examples/raw_cdp_logging.py +++ b/examples/raw_cdp_logging.py @@ -4,7 +4,7 @@ driver = Driver(uc=True, log_cdp=True) try: driver.uc_open_with_reconnect("https://seleniumbase.io/apps/turnstile") - driver.uc_switch_to_frame("iframe") + driver.switch_to_frame("iframe") driver.uc_click("span.mark") driver.sleep(3) pprint(driver.get_log("performance")) diff --git a/examples/raw_form_turnstile.py b/examples/raw_form_turnstile.py index 870c63dccbd..e6597bf3376 100644 --- a/examples/raw_form_turnstile.py +++ b/examples/raw_form_turnstile.py @@ -1,26 +1,7 @@ from seleniumbase import SB - -def open_the_form_turnstile_page(sb): - sb.driver.uc_open_with_reconnect( - "https://seleniumbase.io/apps/form_turnstile", reconnect_time=2.7, - ) - - -def click_turnstile_and_verify(sb): - sb.scroll_to_bottom() - sb.driver.uc_switch_to_frame("iframe") - sb.driver.uc_click("span.mark") - sb.highlight("img#captcha-success", timeout=3.33) - - with SB(uc=True, test=True) as sb: - open_the_form_turnstile_page(sb) - try: - click_turnstile_and_verify(sb) - except Exception: - open_the_form_turnstile_page(sb) - click_turnstile_and_verify(sb) + sb.driver.uc_open_with_reconnect("seleniumbase.io/apps/form_turnstile", 3) sb.press_keys("#name", "SeleniumBase") sb.press_keys("#email", "test@test.test") sb.press_keys("#phone", "1-555-555-5555") @@ -30,6 +11,9 @@ def click_turnstile_and_verify(sb): sb.click('span:contains("9:00 PM")') sb.highlight_click('input[value="AR"] + span') sb.click('input[value="cc"] + span') + sb.switch_to_frame("iframe") + sb.driver.uc_click("span.mark") + sb.highlight("img#captcha-success", timeout=3) sb.highlight_click('button:contains("Request & Pay")') sb.highlight("img#submit-success") sb.highlight('button:contains("Success!")') diff --git a/examples/raw_nopecha.py b/examples/raw_nopecha.py index 53a4f7d2068..62f2ebd3483 100644 --- a/examples/raw_nopecha.py +++ b/examples/raw_nopecha.py @@ -1,14 +1,14 @@ from seleniumbase import SB with SB(uc=True, test=True) as sb: - sb.driver.uc_open_with_reconnect("https://nopecha.com/demo/turnstile", 4) - sb.driver.uc_switch_to_frame("#example-container5 iframe") - sb.driver.uc_click("span.mark", reconnect_time=1) + sb.driver.uc_open_with_reconnect("nopecha.com/demo/turnstile", 4.2) + sb.switch_to_frame("#example-container5 iframe") + sb.driver.uc_click("span.mark") if sb.is_element_visible("#example-container0 iframe"): sb.switch_to_frame("#example-container0 iframe") if not sb.is_element_visible("circle.success-circle"): - sb.driver.uc_click("span.mark", reconnect_time=1) + sb.driver.uc_click("span.mark") sb.switch_to_frame("#example-container0 iframe") sb.assert_element("circle.success-circle") sb.switch_to_parent_frame() diff --git a/examples/raw_turnstile.py b/examples/raw_turnstile.py index bae7cd69b97..59f0c648d02 100644 --- a/examples/raw_turnstile.py +++ b/examples/raw_turnstile.py @@ -3,14 +3,14 @@ def open_the_turnstile_page(sb): sb.driver.uc_open_with_reconnect( - "https://seleniumbase.io/apps/turnstile", reconnect_time=2.7, + "seleniumbase.io/apps/turnstile", reconnect_time=3, ) def click_turnstile_and_verify(sb): - sb.driver.uc_switch_to_frame("iframe") + sb.driver.switch_to_frame("iframe") sb.driver.uc_click("span.mark") - sb.assert_element("img#captcha-success", timeout=3.33) + sb.assert_element("img#captcha-success", timeout=3) with SB(uc=True, test=True) as sb: diff --git a/examples/test_download_files.py b/examples/test_download_files.py index 537a1e5d9e2..941db7ac900 100644 --- a/examples/test_download_files.py +++ b/examples/test_download_files.py @@ -43,6 +43,7 @@ def test_download_files_from_pypi(self): if ( self.browser == "safari" or self.browser == "ie" + or self.browser == "edge" or (self.is_chromium() and self.guest_mode) or (self.is_chromium() and (self.headless or self.headless2)) ): diff --git a/examples/uc_cdp_events.py b/examples/uc_cdp_events.py index bc6cdbd4616..e80f9d95f36 100644 --- a/examples/uc_cdp_events.py +++ b/examples/uc_cdp_events.py @@ -13,15 +13,15 @@ def add_cdp_listener(self): ) def click_turnstile_and_verify(sb): - sb.driver.uc_switch_to_frame("iframe") + sb.switch_to_frame("iframe") sb.driver.uc_click("span.mark") - sb.assert_element("img#captcha-success", timeout=3.33) + sb.assert_element("img#captcha-success", timeout=3) sb.highlight("img#captcha-success", loops=8) def test_display_cdp_events(self): if not (self.undetectable and self.uc_cdp_events): self.get_new_driver(undetectable=True, uc_cdp_events=True) - self.driver.uc_open("https://seleniumbase.io/apps/turnstile") + self.driver.uc_open_with_reconnect("seleniumbase.io/apps/turnstile") self.add_cdp_listener() self.click_turnstile_and_verify() self.sleep(1) From a719c970aeb232db1454f24b9fecc4a123225dd6 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 21 Mar 2024 17:31:44 -0400 Subject: [PATCH 3/4] Refresh Python dependencies --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index d9749094390..2d5ea6b55e6 100755 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ sniffio==1.3.1 h11==0.14.0 outcome==1.3.0.post0 trio==0.22.2;python_version<"3.8" -trio==0.24.0;python_version>="3.8" +trio==0.25.0;python_version>="3.8" trio-websocket==0.11.1 wsproto==1.2.0 selenium==4.11.2;python_version<"3.8" diff --git a/setup.py b/setup.py index acc744a777c..cfce2037099 100755 --- a/setup.py +++ b/setup.py @@ -174,7 +174,7 @@ 'h11==0.14.0', 'outcome==1.3.0.post0', 'trio==0.22.2;python_version<"3.8"', - 'trio==0.24.0;python_version>="3.8"', + 'trio==0.25.0;python_version>="3.8"', 'trio-websocket==0.11.1', 'wsproto==1.2.0', 'selenium==4.11.2;python_version<"3.8"', From 0abea077ae862867cf3fbe3c7f25e0de55b898e0 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 21 Mar 2024 17:32:01 -0400 Subject: [PATCH 4/4] Version 4.24.12 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 4af28a2af2f..bc83d5edb66 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.24.11" +__version__ = "4.24.12"