Merge pull request #3331 from seleniumbase/cdp-mode-patch-18

CDP Mode - Patch 18

Author: mdmintz

examples/cdp_mode/ReadMe.md

@@ -384,6 +384,7 @@ sb.cdp.get_document()
 sb.cdp.get_flattened_document()
 sb.cdp.get_element_attributes(selector)
 sb.cdp.get_element_attribute(selector, attribute)
+sb.cdp.get_attribute(selector, attribute)
 sb.cdp.get_element_html(selector)
 sb.cdp.set_locale(locale)
 sb.cdp.set_attributes(selector, attribute, value)
@@ -463,6 +464,7 @@ element.type(text)
 element.get_position()
 element.get_html()
 element.get_js_attributes()
+element.get_attribute(attribute)
 ```
 
 --------

examples/cdp_mode/raw_fingerprint.py

@@ -1,6 +1,6 @@
 from seleniumbase import SB
 
-with SB(uc=True, test=True, incognito=True) as sb:
+with SB(uc=True, test=True) as sb:
     url = "https://demo.fingerprint.com/playground"
     sb.activate_cdp_mode(url)
     sb.sleep(1)

examples/cdp_mode/raw_publication.py

@@ -0,0 +1,20 @@
+from seleniumbase import SB
+
+with SB(uc=True, test=True, locale_code="en", ad_block=True) as sb:
+    url = "https://www.researchgate.net/search/publication"
+    sb.activate_cdp_mode(url)
+    sb.sleep(2)
+    if sb.cdp.is_element_visible("p.spacer-bottom"):
+        sb.uc_gui_click_captcha()
+    shadow_head = "div.main-content div"
+    if sb.is_element_present(shadow_head):
+        sb.cdp.gui_click_element(shadow_head)
+        sb.sleep(1.5)
+    sb.cdp.remove_elements('div[class*="ad-container"]')
+    sb.cdp.remove_elements("div.lite-page-ad")
+    sb.sleep(0.5)
+    sb.assert_text("Discover the world's scientific knowledge")
+    sb.highlight('h1[class*="nova"]')
+    sb.highlight('input[name="q"]')
+    sb.highlight("a.menu-item.selected")
+    sb.sleep(1)

examples/cdp_mode/raw_res_sb.py

@@ -1,4 +1,4 @@
-"""Using CDP.network.ResponseReceived and CDP.network.RequestWillBeSent."""
+"""Using CDP.network.RequestWillBeSent and CDP.network.ResponseReceived."""
 import colorama
 import mycdp
 import sys

examples/raw_recaptcha.py

@@ -1,16 +1,18 @@
 from seleniumbase import SB
 
-with SB(uc=True, test=True) as sb:
+with SB(uc=True, test=True, incognito=True) as sb:
     url = "https://seleniumbase.io/apps/recaptcha"
     sb.activate_cdp_mode(url)
+    sb.sleep(1)
     sb.uc_gui_handle_captcha()  # Try with TAB + SPACEBAR
     sb.assert_element("img#captcha-success", timeout=3)
     sb.set_messenger_theme(location="top_left")
     sb.post_message("SeleniumBase wasn't detected", duration=3)
 
-with SB(uc=True, test=True) as sb:
+with SB(uc=True, test=True, incognito=True) as sb:
     url = "https://seleniumbase.io/apps/recaptcha"
     sb.activate_cdp_mode(url)
+    sb.sleep(1)
     sb.uc_gui_click_captcha('iframe[src*="/recaptcha/"]')
     sb.assert_element("img#captcha-success", timeout=3)
     sb.set_messenger_theme(location="top_left")

mkdocs_build/requirements.txt

@@ -6,7 +6,7 @@ pymdown-extensions>=10.12
 pipdeptree>=2.24.0
 python-dateutil>=2.8.2
 Markdown==3.7
-markdown2==2.5.1
+markdown2==2.5.2
 MarkupSafe==3.0.2
 Jinja2==3.1.4
 click==8.1.7

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.33.7"
+__version__ = "4.33.8"

seleniumbase/core/browser_launcher.py

@@ -658,6 +658,7 @@ def uc_open_with_cdp_mode(driver, url=None):
     cdp.get_window = CDPM.get_window
     cdp.get_element_attributes = CDPM.get_element_attributes
     cdp.get_element_attribute = CDPM.get_element_attribute
+    cdp.get_attribute = CDPM.get_attribute
     cdp.get_element_html = CDPM.get_element_html
     cdp.get_element_rect = CDPM.get_element_rect
     cdp.get_element_size = CDPM.get_element_size
@@ -2321,7 +2322,6 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-ipc-flooding-protection")
     chrome_options.add_argument("--disable-password-generation")
     chrome_options.add_argument("--disable-domain-reliability")
-    chrome_options.add_argument("--disable-component-update")
     chrome_options.add_argument("--disable-breakpad")
     included_disabled_features = []
     included_disabled_features.append("OptimizationHints")
@@ -4071,7 +4071,6 @@ def get_local_driver(
         edge_options.add_argument("--disable-ipc-flooding-protection")
         edge_options.add_argument("--disable-password-generation")
         edge_options.add_argument("--disable-domain-reliability")
-        edge_options.add_argument("--disable-component-update")
         edge_options.add_argument("--disable-breakpad")
         included_disabled_features = []
         included_disabled_features.append("OptimizationHints")

seleniumbase/core/sb_cdp.py

@@ -83,6 +83,9 @@ def __add_sync_methods(self, element):
         element.get_position = lambda: self.__get_position(element)
         element.get_html = lambda: self.__get_html(element)
         element.get_js_attributes = lambda: self.__get_js_attributes(element)
+        element.get_attribute = (
+            lambda attribute: self.__get_attribute(element, attribute)
+        )
         return element
 
     def get(self, url):
@@ -440,6 +443,12 @@ def __get_js_attributes(self, element):
             self.loop.run_until_complete(element.get_js_attributes_async())
         )
 
+    def __get_attribute(self, element, attribute):
+        try:
+            return element.get_js_attributes()[attribute]
+        except Exception:
+            return None
+
     def __get_x_scroll_offset(self):
         x_scroll_offset = self.loop.run_until_complete(
             self.page.evaluate("window.pageXOffset")
@@ -1013,8 +1022,10 @@ def get_element_attributes(self, selector):
         )
 
     def get_element_attribute(self, selector, attribute):
-        attributes = self.get_element_attributes(selector)
-        return attributes[attribute]
+        return self.get_element_attributes(selector)[attribute]
+
+    def get_attribute(self, selector, attribute):
+        return self.find_element(selector).get_attribute(attribute)
 
     def get_element_html(self, selector):
         selector = self.__convert_to_css_if_xpath(selector)
@@ -1774,31 +1785,26 @@ def scroll_to_y(self, y):
         with suppress(Exception):
             self.loop.run_until_complete(self.page.evaluate(js_code))
             self.loop.run_until_complete(self.page.wait())
-        self.__add_light_pause()
 
     def scroll_to_top(self):
         js_code = "window.scrollTo(0, 0);"
         with suppress(Exception):
             self.loop.run_until_complete(self.page.evaluate(js_code))
             self.loop.run_until_complete(self.page.wait())
-        self.__add_light_pause()
 
     def scroll_to_bottom(self):
         js_code = "window.scrollTo(0, 10000);"
         with suppress(Exception):
             self.loop.run_until_complete(self.page.evaluate(js_code))
             self.loop.run_until_complete(self.page.wait())
-        self.__add_light_pause()
 
     def scroll_up(self, amount=25):
         self.loop.run_until_complete(self.page.scroll_up(amount))
         self.loop.run_until_complete(self.page.wait())
-        self.__add_light_pause()
 
     def scroll_down(self, amount=25):
         self.loop.run_until_complete(self.page.scroll_down(amount))
         self.loop.run_until_complete(self.page.wait())
-        self.__add_light_pause()
 
     def save_screenshot(self, name, folder=None, selector=None):
         filename = name

seleniumbase/core/settings_parser.py

@@ -71,6 +71,8 @@ def set_settings(settings_file):
             settings.LARGE_TIMEOUT = override_settings[key]
         elif key == "EXTREME_TIMEOUT":
             settings.EXTREME_TIMEOUT = override_settings[key]
+        elif key == "PAGE_LOAD_TIMEOUT":
+            settings.PAGE_LOAD_TIMEOUT = override_settings[key]
         elif key == "ARCHIVE_EXISTING_LOGS":
             settings.ARCHIVE_EXISTING_LOGS = override_settings[key]
         elif key == "ARCHIVE_EXISTING_DOWNLOADS":

seleniumbase/fixtures/base_case.py

@@ -6051,10 +6051,8 @@ def highlight(
         scroll - the option to scroll to the element first (Default: True)
         timeout - the time to wait for the element to appear """
         self.__check_scope()
-        if self.__is_cdp_swap_needed() and ":contains(" not in selector:
-            self.cdp.highlight(selector)
-            return
-        self._check_browser()
+        if not self.__is_cdp_swap_needed():
+            self._check_browser()
         self.__skip_if_esc()
         if isinstance(selector, WebElement):
             self.__highlight_element(selector, loops=loops, scroll=scroll)
@@ -13860,7 +13858,8 @@ def __slow_scroll_to_element(self, element):
             js_utils.scroll_to_element(self.driver, element)
 
     def __highlight_with_js(self, selector, loops, o_bs):
-        self.wait_for_ready_state_complete()
+        if not self.__is_cdp_swap_needed():
+            self.wait_for_ready_state_complete()
         js_utils.highlight_with_js(self.driver, selector, loops, o_bs)
 
     def __highlight_element_with_js(self, element, loops, o_bs):

seleniumbase/undetected/cdp_driver/config.py

@@ -125,7 +125,6 @@ def __init__(
             "--deny-permission-prompts",
             "--disable-infobars",
             "--disable-breakpad",
-            "--disable-component-update",
             "--disable-prompt-on-repost",
             "--disable-password-generation",
             "--disable-ipc-flooding-protection",

seleniumbase/undetected/cdp_driver/element.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 import asyncio
-import json
 import logging
 import pathlib
 import secrets
@@ -387,18 +386,16 @@ async def click_async(self):
 
     async def get_js_attributes_async(self):
         return ContraDict(
-            json.loads(
-                await self.apply(
-                    """
-            function (e) {
-                let o = {}
-                for(let k in e){
-                    o[k] = e[k]
+            await self.apply(
+                """
+                function (e) {
+                    let o = {}
+                    for(let k in e){
+                        o[k] = e[k]
+                    }
+                    return o
                 }
-                return JSON.stringify(o)
-            }
-            """
-                )
+                """
             )
         )