Merge pull request #3996 from seleniumbase/cdp-mode-patch-59

CDP Mode: Patch 59

Author: mdmintz

README.md

@@ -99,6 +99,7 @@ with SB(uc=True, test=True, locale="en") as sb:
     sb.activate_cdp_mode(url)
     sb.sleep(2.2)
     sb.uc_gui_click_captcha()
+    # (The rest is for testing and demo purposes)
     sb.assert_text("Username", '[for="user_login"]', timeout=3)
     sb.assert_element('label[for="user_login"]')
     sb.highlight('button:contains("Sign in")')

examples/boilerplates/samples/sb_swag_test.py

@@ -1,16 +1,18 @@
 """Classic Page Object Model with the "sb" fixture."""
+from seleniumbase import BaseCase
+BaseCase.main(__name__, __file__)
 
 
 class LoginPage:
-    def login_to_swag_labs(self, sb, username):
+    def login_to_swag_labs(self, sb: BaseCase, username):
         sb.open("https://www.saucedemo.com")
         sb.type("#user-name", username)
         sb.type("#password", "secret_sauce")
         sb.click('input[type="submit"]')
 
 
 class MyTests:
-    def test_swag_labs_login(self, sb):
+    def test_swag_labs_login(self, sb: BaseCase):
         LoginPage().login_to_swag_labs(sb, "standard_user")
         sb.assert_element("div.inventory_list")
         sb.assert_element('div:contains("Sauce Labs Backpack")')

examples/boilerplates/samples/swag_labs_test.py

@@ -4,7 +4,7 @@
 
 
 class LoginPage:
-    def login_to_swag_labs(self, sb, username):
+    def login_to_swag_labs(self, sb: BaseCase, username):
         sb.open("https://www.saucedemo.com")
         sb.type("#user-name", username)
         sb.type("#password", "secret_sauce")

examples/cdp_mode/ReadMe.md

@@ -45,28 +45,27 @@
 
 That disconnects WebDriver from Chrome (which prevents detection), and gives you access to `sb.cdp` methods (which don't trigger anti-bot checks).
 
-Simple example: ([SeleniumBase/examples/cdp_mode/raw_gitlab.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_gitlab.py))
+Simple example from [SeleniumBase/examples/cdp_mode/raw_gitlab.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_gitlab.py):
 
 ```python
 from seleniumbase import SB
 
 with SB(uc=True, test=True, locale="en") as sb:
     url = "https://gitlab.com/users/sign_in"
     sb.activate_cdp_mode(url)
-    sb.sleep(1)
+    sb.sleep(2.2)
     sb.uc_gui_click_captcha()
-    sb.sleep(2)
 ```
 
 <img src="https://seleniumbase.github.io/other/cf_sec.jpg" title="SeleniumBase" width="332"> <img src="https://seleniumbase.github.io/other/gitlab_bypass.png" title="SeleniumBase" width="288">
 
-(If the CAPTCHA wasn't bypassed automatically, then `sb.uc_gui_click_captcha()` gets the job done.)
+(If the CAPTCHA wasn't bypassed automatically when going to the URL, then `sb.uc_gui_click_captcha()` gets the job done with a mouse click from [PyAutoGUI](https://github.com/asweigart/pyautogui).)
 
-Note that `PyAutoGUI` is an optional dependency. If calling a method that uses it when not already installed, then `SeleniumBase` installs `PyAutoGUI` at run-time.
+ℹ️ Note that `PyAutoGUI` is an optional dependency. If calling a method that uses it when not already installed, then `SeleniumBase` installs `PyAutoGUI` at run-time.
 
 --------
 
-For some Cloudflare CAPTCHAs that appear within websites, you may need to use `sb.cdp.gui_click_element(selector)` instead (if the Turnstile wasn't bypassed automatically). Example: ([SeleniumBase/examples/cdp_mode/raw_planetmc.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_planetmc.py))
+You can also use `sb.cdp.gui_click_element(selector)` to click on elements using `PyAutoGUI`. (This is useful when clicking inside `#shadow-root`.) Example:
 
 ```python
 from seleniumbase import SB
@@ -86,17 +85,19 @@ Eg. `sb.cdp.gui_click_element("#turnstile-widget div")`
 
 <img src="https://seleniumbase.github.io/other/above_shadow.png" title="SeleniumBase" width="480">
 
+In most cases, `sb.uc_gui_click_captcha()` is good enough for CF Turnstiles without needing `sb.cdp.gui_click_element(selector)`. (See [SeleniumBase/examples/cdp_mode/raw_planetmc.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_planetmc.py))
+
 --------
 
 ### 🐙 Here are a few common `sb.cdp` methods:
 
 * `sb.cdp.click(selector)`  (Uses the CDP API to click)
-* `sb.cdp.click_if_visible(selector)`
+* `sb.cdp.click_if_visible(selector)`  (Click if visible)
 * `sb.cdp.gui_click_element(selector)`  (Uses `PyAutoGUI`)
-* `sb.cdp.type(selector, text)`
+* `sb.cdp.type(selector, text)`  (Type text into a selector)
 * `sb.cdp.press_keys(selector, text)`  (Human-speed `type`)
-* `sb.cdp.select_all(selector)`
-* `sb.cdp.get_text(selector)`
+* `sb.cdp.select_all(selector)`  (Returns matching elements)
+* `sb.cdp.get_text(selector)`  (Returns the element's text)
 
 Methods that start with `sb.cdp.gui` use `PyAutoGUI` for interaction.
 
@@ -161,18 +162,12 @@ with SB(uc=True, test=True, locale="en", ad_block=True) as sb:
     sb.sleep(2)
     sb.cdp.highlight_overlay("div.pokemon-ability-info")
     sb.sleep(2)
-    sb.cdp.click('a[href="https://www.pokemon.com/us/play-pokemon/"]')
-    sb.sleep(0.6)
-    sb.cdp.click('h3:contains("Find an Event")')
-    location = "Concord, MA, USA"
-    sb.cdp.type('input[data-testid="location-search"]', location)
-    sb.sleep(1.5)
-    sb.cdp.click("div.autocomplete-dropdown-container div.suggestion-item")
-    sb.sleep(0.6)
-    sb.cdp.click('img[alt="search-icon"]')
-    sb.sleep(2)
-    events = sb.cdp.select_all('div[data-testid="event-name"]')
-    print("*** Pokemon events near %s: ***" % location)
+    sb.cdp.open("https://events.pokemon.com/EventLocator/")
+    sb.sleep(3)
+    sb.cdp.click('button span:contains("Premier Events")')
+    sb.sleep(1)
+    events = sb.cdp.select_all('div[class="event-info"]')
+    print("*** Upcoming Premier Events for Pokémon: ***")
     for event in events:
         print("* " + event.text)
     sb.sleep(2)
@@ -371,6 +366,9 @@ sb.cdp.select(selector, timeout=None)
 sb.cdp.select_all(selector, timeout=None)
 sb.cdp.find_elements(selector, timeout=None)
 sb.cdp.find_visible_elements(selector, timeout=None)
+sb.cdp.click(selector, timeout=None)
+sb.cdp.click_if_visible(selector)
+sb.cdp.click_visible_elements(selector, limit=0)
 sb.cdp.click_nth_element(selector, number)
 sb.cdp.click_nth_visible_element(selector, number)
 sb.cdp.click_link(link_text)
@@ -391,10 +389,7 @@ sb.cdp.bring_active_window_to_front()
 sb.cdp.bring_to_front()
 sb.cdp.get_active_element()
 sb.cdp.get_active_element_css()
-sb.cdp.click(selector, timeout=None)
 sb.cdp.click_active_element()
-sb.cdp.click_if_visible(selector)
-sb.cdp.click_visible_elements(selector, limit=0)
 sb.cdp.mouse_click(selector, timeout=None)
 sb.cdp.nested_click(parent_selector, selector)
 sb.cdp.get_nested_element(parent_selector, selector)
@@ -488,6 +483,7 @@ sb.cdp.is_exact_text_visible(text, selector="body")
 sb.cdp.wait_for_text(text, selector="body", timeout=None)
 sb.cdp.wait_for_text_not_visible(text, selector="body", timeout=None)
 sb.cdp.wait_for_element_visible(selector, timeout=None)
+sb.cdp.wait_for_element(selector, timeout=None)
 sb.cdp.wait_for_element_not_visible(selector, timeout=None)
 sb.cdp.wait_for_element_absent(selector, timeout=None)
 sb.cdp.wait_for_any_of_elements_visible(*args, **kwargs)
@@ -524,10 +520,66 @@ sb.cdp.print_to_pdf(name, folder=None)
 sb.cdp.save_as_pdf(name, folder=None)
 ```
 
+ℹ️ When available, calling `sb.METHOD()` redirects to `sb.cdp.METHOD()` because regular SB methods automatically call their CDP Mode counterparts to maintain stealth when CDP Mode is active.
+
+--------
+
+### 🐙 <b translate="no">Pure CDP Mode</b> (<code translate="no">sb_cdp</code>)
+
+<b translate="no">Pure CDP Mode</b> doesn't use WebDriver for anything. The browser is launched using CDP, and all browser actions are performed using CDP (or <code>PyAutoGUI</code>). Initialization:
+
+```python
+from seleniumbase import sb_cdp
+
+sb = sb_cdp.Chrome(url=None, **kwargs)
+```
+
+<b translate="no">Pure CDP Mode</b> includes all methods from regular CDP Mode, except that they're called directly from <code>sb</code> instead of <code>sb.cdp</code>. Eg: <code>sb.gui_click_captcha()</code>. To quit a CDP-launched browser, use `sb.driver.stop()`.
+
+Basic example from [SeleniumBase/examples/cdp_mode/raw_cdp_turnstile.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_cdp_turnstile.py):
+
+```python
+from seleniumbase import sb_cdp
+
+url = "https://seleniumbase.io/apps/turnstile"
+sb = sb_cdp.Chrome(url)
+sb.gui_click_captcha()
+sb.sleep(2)
+sb.driver.stop()
+```
+
+Another example: ([SeleniumBase/examples/cdp_mode/raw_cdp_methods.py](https://github.com/seleniumbase/SeleniumBase/blob/master/examples/cdp_mode/raw_cdp_methods.py))
+
+```python
+from seleniumbase import sb_cdp
+
+url = "https://seleniumbase.io/demo_page"
+sb = sb_cdp.Chrome(url)
+sb.press_keys("input", "Text")
+sb.highlight("button")
+sb.type("textarea", "Here are some words")
+sb.click("button")
+sb.set_value("input#mySlider", "100")
+sb.click_visible_elements("input.checkBoxClassB")
+sb.select_option_by_text("#mySelect", "Set to 75%")
+sb.gui_hover_and_click("#myDropdown", "#dropOption2")
+sb.gui_click_element("#checkBox1")
+sb.gui_drag_and_drop("img#logo", "div#drop2")
+sb.nested_click("iframe#myFrame3", ".fBox")
+sb.sleep(2)
+sb.driver.stop()
+```
+
+ℹ️ Even if you don't call `sb.driver.stop()`, the browser still quits after the script goes out-of-scope.
+
 --------
 
 ### 🐙 <b translate="no">CDP Mode</b> WebElement API / Methods
 
+After finding an element in CDP Mode, you can access `WebElement` methods:
+
+(Eg. After `element = sb.find_element(selector)`)
+
 ```python
 element.clear_input()
 element.click()

examples/cdp_mode/raw_ahrefs.py

@@ -8,7 +8,9 @@
     sb.type(input_field, "github.com/seleniumbase/SeleniumBase")
     sb.cdp.scroll_down(36)
     sb.click(submit_button)
+    sb.sleep(1)
     sb.uc_gui_click_captcha()
+    sb.sleep(3)
     sb.wait_for_text_not_visible("Checking", timeout=15)
     sb.click_if_visible('button[data-cky-tag="close-button"]')
     sb.highlight('p:contains("github.com/seleniumbase/SeleniumBase")')

examples/cdp_mode/raw_copilot.py

@@ -0,0 +1,34 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, guest=True) as sb:
+    url = "https://copilot.microsoft.com/"
+    sb.activate_cdp_mode(url)
+    textarea = "textarea#userInput"
+    sb.wait_for_element(textarea)
+    sb.sleep(1.5)
+    sb.click_if_visible('[aria-label="Dismiss"]')
+    sb.sleep(0.5)
+    sb.click('button[data-testid*="chat-mode-"]')
+    sb.sleep(1.1)
+    sb.click('button[title="Think Deeper"]')
+    sb.sleep(1.1)
+    query = "How to migrate from Playwright to SeleniumBase?"
+    sb.press_keys(textarea, query)
+    sb.sleep(1.1)
+    sb.click('button[data-testid="submit-button"]')
+    sb.sleep(2.5)
+    sb.uc_gui_click_captcha()
+    sb.sleep(2.5)
+    sb.uc_gui_click_captcha()
+    sb.sleep(2.5)
+    stop_button = '[data-testid="stop-button"]'
+    thumbs_up = 'button[data-testid*="-thumbs-up-"]'
+    sb.wait_for_element_absent(stop_button, timeout=30)
+    sb.wait_for_element(thumbs_up, timeout=30)
+    sb.sleep(0.5)
+    sb.click('button[data-testid*="scroll-to-bottom"]')
+    sb.sleep(1.5)
+    folder = "downloaded_files"
+    file_name = "copilot_results.html"
+    sb.save_page_source(file_name, folder)
+    print('"./%s/%s" was saved!' % (folder, file_name))

examples/cdp_mode/raw_gitlab.py

@@ -5,6 +5,7 @@
     sb.activate_cdp_mode(url)
     sb.sleep(2.2)
     sb.uc_gui_click_captcha()
+    # (The rest is for testing and demo purposes)
     sb.assert_text("Username", '[for="user_login"]', timeout=3)
     sb.assert_element('label[for="user_login"]')
     sb.highlight('button:contains("Sign in")')

examples/cdp_mode/raw_multi_captcha.py

@@ -8,7 +8,7 @@
 def main(url):
     sb = sb_cdp.Chrome(url, lang="en")
     sb.set_window_rect(randint(4, 680), randint(8, 380), 840, 520)
-    sb.sleep(2.2)
+    sb.sleep(2)
     sb.gui_click_captcha()
     sb.sleep(2)
     sb.driver.quit()

examples/cdp_mode/raw_planetmc.py

@@ -4,5 +4,5 @@
     url = "www.planetminecraft.com/account/sign_in/"
     sb.activate_cdp_mode(url)
     sb.sleep(2)
-    sb.cdp.gui_click_element("#turnstile-widget div")
+    sb.uc_gui_click_captcha()
     sb.sleep(2)

examples/cdp_mode/raw_turnstile.py

@@ -0,0 +1,9 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True) as sb:
+    url = "https://seleniumbase.io/apps/turnstile"
+    sb.activate_cdp_mode(url)
+    sb.uc_gui_click_captcha()
+    sb.assert_element("img#captcha-success", timeout=3)
+    sb.set_messenger_theme(location="top_left")
+    sb.post_message("SeleniumBase wasn't detected", duration=3)