Merge pull request #2577 from seleniumbase/better-pytest-collection-and-more

mdmintz · web-flow · commit e7da76058346 · 2024-03-07T20:53:01.000-05:00
Improve pytest collection and more
diff --git a/examples/presenter/multi_uc.py b/examples/presenter/multi_uc.py
@@ -7,17 +7,18 @@
 
 @pytest.mark.parametrize("", [[]] * 3)
 def test_multi_threaded(sb):
-    sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
+    url = "https://gitlab.com/users/sign_in"
+    sb.driver.uc_open_with_reconnect(url, 3)
     sb.set_window_rect(randint(0, 755), randint(38, 403), 700, 500)
     try:
-        sb.assert_text("Discord Bots", "h1", timeout=2)
-        sb.post_message("Selenium wasn't detected!", duration=4)
+        sb.assert_text("Username", '[for="user_login"]', timeout=3)
+        sb.post_message("SeleniumBase wasn't detected", duration=4)
         sb._print("\n Success! Website did not detect Selenium! ")
     except Exception:
-        sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
+        sb.driver.uc_open_with_reconnect(url, 3)
         try:
-            sb.assert_text("Discord Bots", "h1", timeout=2)
-            sb.post_message("Selenium wasn't detected!", duration=4)
+            sb.assert_text("Username", '[for="user_login"]', timeout=3)
+            sb.post_message("SeleniumBase wasn't detected", duration=4)
             sb._print("\n Success! Website did not detect Selenium! ")
         except Exception:
             sb.fail('Selenium was detected! Try using: "pytest --uc"')
diff --git a/examples/presenter/uc_presentation.py b/examples/presenter/uc_presentation.py
@@ -28,19 +28,16 @@ def test_presentation(self):
         self.begin_presentation(filename="uc_presentation.html")
 
         self.get_new_driver(undetectable=True)
+        url = "https://gitlab.com/users/sign_in"
         try:
-            self.driver.uc_open_with_reconnect(
-                "https://top.gg/", reconnect_time=4
-            )
+            self.driver.uc_open_with_reconnect(url, reconnect_time=3)
             try:
-                self.assert_text("Discord Bots", "h1", timeout=3)
-                self.post_message("Selenium wasn't detected!", duration=4)
+                self.assert_text("Username", '[for="user_login"]', timeout=3)
+                self.post_message("SeleniumBase wasn't detected", duration=4)
             except Exception:
-                self.driver.uc_open_with_reconnect(
-                    "https://top.gg/", reconnect_time=5
-                )
-                self.assert_text("Discord Bots", "h1", timeout=2)
-                self.post_message("Selenium wasn't detected!", duration=4)
+                self.driver.uc_open_with_reconnect(url, reconnect_time=4)
+                self.assert_text("Username", '[for="user_login"]', timeout=3)
+                self.post_message("SeleniumBase wasn't detected", duration=4)
         finally:
             self.quit_extra_driver()
 
diff --git a/examples/raw_call.py b/examples/raw_call.py
@@ -4,5 +4,6 @@
 import pytest
 import subprocess
 
-pytest.main(["test_coffee_cart.py", "--chrome", "-v"])
-subprocess.call(["pytest", "test_mfa_login.py", "--chrome", "-v"])
+if __name__ == "__main__":
+    pytest.main(["test_coffee_cart.py", "--chrome", "-v"])
+    subprocess.call(["pytest", "test_mfa_login.py", "--chrome", "-v"])
diff --git a/examples/raw_multi_drivers.py b/examples/raw_multi_drivers.py
@@ -0,0 +1,27 @@
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from random import randint, seed
+from seleniumbase import Driver
+sys.argv.append("-n")  # Tell SeleniumBase to do thread-locking as needed
+
+
+def launch_driver(url):
+    seed(len(threading.enumerate()))  # Random seed for browser placement
+    driver = Driver()
+    try:
+        driver.set_window_rect(randint(4, 720), randint(8, 410), 700, 500)
+        driver.get(url=url)
+        if driver.is_element_visible("h1"):
+            driver.highlight("h1", loops=9)
+        else:
+            driver.sleep(2.2)
+    finally:
+        driver.quit()
+
+
+if __name__ == "__main__":
+    urls = ['https://seleniumbase.io/demo_page' for i in range(4)]
+    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
+        for url in urls:
+            executor.submit(launch_driver, url)
diff --git a/examples/raw_turnstile.py b/examples/raw_turnstile.py
@@ -21,4 +21,4 @@ def click_turnstile_and_verify(sb):
         open_the_turnstile_page(sb)
         click_turnstile_and_verify(sb)
     sb.set_messenger_theme(location="top_left")
-    sb.post_message("Selenium wasn't detected!", duration=3)
+    sb.post_message("SeleniumBase wasn't detected", duration=3)
diff --git a/examples/raw_uc_mode.py b/examples/raw_uc_mode.py
@@ -2,10 +2,10 @@
 from seleniumbase import SB
 
 with SB(uc=True, test=True) as sb:
-    sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
-    if not sb.is_text_visible("Discord Bots", "h1"):
-        sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
-    sb.assert_text("Discord Bots", "h1", timeout=3)
-    sb.highlight("h1", loops=3)
-    sb.set_messenger_theme(location="top_center")
-    sb.post_message("Selenium wasn't detected!", duration=3)
+    url = "https://gitlab.com/users/sign_in"
+    sb.driver.uc_open_with_reconnect(url, 3)
+    if not sb.is_text_visible("Username", '[for="user_login"]'):
+        sb.driver.uc_open_with_reconnect(url, 4)
+    sb.assert_text("Username", '[for="user_login"]', timeout=3)
+    sb.highlight('label[for="user_login"]', loops=3)
+    sb.post_message("SeleniumBase wasn't detected", duration=4)
diff --git a/examples/verify_undetected.py b/examples/verify_undetected.py
@@ -7,13 +7,13 @@
 
 class UndetectedTest(BaseCase):
     def test_browser_is_undetected(self):
+        url = "https://gitlab.com/users/sign_in"
         if not self.undetectable:
             self.get_new_driver(undetectable=True)
-        self.driver.uc_open_with_reconnect("https://top.gg/", 5)
-        if not self.is_text_visible("Discord Bots", "h1"):
+        self.driver.uc_open_with_reconnect(url, 3)
+        if not self.is_text_visible("Username", '[for="user_login"]'):
             self.get_new_driver(undetectable=True)
-            self.driver.uc_open_with_reconnect("https://top.gg/", 5)
-        self.assert_text("Discord Bots", "h1", timeout=3)
-        self.set_messenger_theme(location="top_center")
-        self.post_message("Selenium wasn't detected!", duration=2.8)
+            self.driver.uc_open_with_reconnect(url, 4)
+        self.assert_text("Username", '[for="user_login"]', timeout=3)
+        self.post_message("SeleniumBase wasn't detected", duration=4)
         self._print("\n Success! Website did not detect Selenium! ")
diff --git a/help_docs/uc_mode.md b/help_docs/uc_mode.md
@@ -25,7 +25,7 @@
 from seleniumbase import Driver
 
 driver = Driver(uc=True)
-driver.uc_open_with_reconnect("https://top.gg/", 6)
+driver.uc_open_with_reconnect("https://gitlab.com/users/sign_in", 3)
 driver.quit()
 ```
 
@@ -35,7 +35,7 @@ driver.quit()
 from seleniumbase import SB
 
 with SB(uc=True) as sb:
-    sb.driver.uc_open_with_reconnect("https://top.gg/", 6)
+    sb.driver.uc_open_with_reconnect("https://gitlab.com/users/sign_in", 3)
 ```
 
 👤 Here's a longer example, which includes a retry if the CAPTCHA isn't bypassed on the first attempt:
@@ -44,13 +44,13 @@ with SB(uc=True) as sb:
 from seleniumbase import SB
 
 with SB(uc=True, test=True) as sb:
-    sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
-    if not sb.is_text_visible("Discord Bots", "h1"):
-        sb.driver.uc_open_with_reconnect("https://top.gg/", 5)
-    sb.assert_text("Discord Bots", "h1", timeout=3)
-    sb.highlight("h1", loops=3)
-    sb.set_messenger_theme(location="top_center")
-    sb.post_message("Selenium wasn't detected!", duration=3)
+    url = "https://gitlab.com/users/sign_in"
+    sb.driver.uc_open_with_reconnect(url, 3)
+    if not sb.is_text_visible("Username", '[for="user_login"]'):
+        sb.driver.uc_open_with_reconnect(url, 4)
+    sb.assert_text("Username", '[for="user_login"]', timeout=3)
+    sb.highlight('label[for="user_login"]', loops=3)
+    sb.post_message("SeleniumBase wasn't detected", duration=4)
 ```
 
 👤 Here's an example where clicking the checkbox is required, even for humans: (Commonly seen with forms that are CAPTCHA-protected.)
@@ -76,7 +76,7 @@ with SB(uc=True, test=True) as sb:
         open_the_turnstile_page(sb)
         click_turnstile_and_verify(sb)
     sb.set_messenger_theme(location="top_left")
-    sb.post_message("Selenium wasn't detected!", duration=3)
+    sb.post_message("SeleniumBase wasn't detected", duration=3)
 ```
 
 ### 👤 Here are some examples that use UC Mode:
@@ -117,8 +117,9 @@ driver.default_get(url)  # Faster, but Selenium can be detected
 👤 Here are some examples of using those special UC Mode methods: (Use `self.driver` for `BaseCase` formats. Use `sb.driver` for `SB()` formats):
 
 ```python
-driver.uc_open_with_reconnect("https://top.gg/", reconnect_time=5)
-driver.uc_open_with_reconnect("https://top.gg/", 5)
+url = "https://gitlab.com/users/sign_in"
+driver.uc_open_with_reconnect(url, reconnect_time=3)
+driver.uc_open_with_reconnect(url, 3)
 
 driver.reconnect(5)
 driver.reconnect(timeout=5)
@@ -127,8 +128,9 @@ driver.reconnect(timeout=5)
 👤 You can also set the `reconnect_time` / `timeout` to `"breakpoint"` as a valid option. This allows the user to perform manual actions (until typing `c` and pressing ENTER to continue from the breakpoint):
 
 ```python
-driver.uc_open_with_reconnect("https://top.gg/", reconnect_time="breakpoint")
-driver.uc_open_with_reconnect("https://top.gg/", "breakpoint")
+url = "https://gitlab.com/users/sign_in"
+driver.uc_open_with_reconnect(url, reconnect_time="breakpoint")
+driver.uc_open_with_reconnect(url, "breakpoint")
 
 driver.reconnect(timeout="breakpoint")
 driver.reconnect("breakpoint")
@@ -150,3 +152,37 @@ with SB(uc=True) as sb:
 ```
 
 (If you remain undetected while loading the page and performing manual actions, then you know you can create a working script once you swap the breakpoint with a time, and add special methods like `uc_click` as needed.)
+
+👤 <b>Multithreaded UC Mode:</b>
+
+If you're using `pytest` for multithreaded UC Mode (which requires using one of the `pytest` [syntax formats](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/syntax_formats.md)), then all you have to do is set the number of threads when your script runs. (`-n NUM`) Eg:
+
+```bash
+pytest --uc -n 4
+```
+
+(Then `pytest-xdist` is automatically used to spin up and process the threads.)
+
+If you don't want to use `pytest` for multithreading, then you'll need to do a little more work. That involves using a different multithreading library, (eg. `concurrent.futures`), and making sure that thread-locking is done correctly for processes that share resources. To handle that thread-locking, include `sys.argv.append("-n")` in your SeleniumBase file.
+
+Here's a sample script that uses `concurrent.futures` for spinning up multiple processes:
+
+```python
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from seleniumbase import Driver
+sys.argv.append("-n")  # Tell SeleniumBase to do thread-locking as needed
+
+def launch_driver(url):
+    driver = Driver(uc=True)
+    try:
+        driver.get(url=url)
+        driver.sleep(2)
+    finally:
+        driver.quit()
+
+urls = ['https://seleniumbase.io/demo_page' for i in range(3)]
+with ThreadPoolExecutor(max_workers=len(urls)) as executor:
+    for url in urls:
+        executor.submit(launch_driver, url)
+```
diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt
@@ -2,8 +2,8 @@
 # Minimum Python version: 3.8 (for generating docs only)
 
 regex>=2023.12.25
-pymdown-extensions>=10.7
-pipdeptree>=2.15.1
+pymdown-extensions>=10.7.1
+pipdeptree>=2.16.1
 python-dateutil>=2.8.2
 Markdown==3.5.2
 markdown2==2.4.13
@@ -20,7 +20,7 @@ lxml==5.1.0
 pyquery==2.0.0
 readtime==3.0.0
 mkdocs==1.5.3
-mkdocs-material==9.5.12
+mkdocs-material==9.5.13
 mkdocs-exclude-search==0.6.6
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.3.1
diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py
@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.24.3"
+__version__ = "4.24.4"
diff --git a/seleniumbase/core/sb_driver.py b/seleniumbase/core/sb_driver.py
@@ -147,7 +147,7 @@ def get_user_agent(self, *args, **kwargs):
         return js_utils.get_user_agent(self.driver, *args, **kwargs)
 
     def highlight(self, *args, **kwargs):
-        w_args = kwargs
+        w_args = kwargs.copy()
         if "loops" in w_args:
             w_args.pop("loops")
         element = page_actions.wait_for_element(self.driver, *args, **w_args)
diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py
@@ -127,11 +127,39 @@ def Driver(
     wire=None,  # Shortcut / Duplicate of "use_wire".
     pls=None,  # Shortcut / Duplicate of "page_load_strategy".
 ):
+    from seleniumbase import config as sb_config
     from seleniumbase.fixtures import constants
     from seleniumbase.fixtures import shared_utils
 
     sys_argv = sys.argv
     arg_join = " ".join(sys_argv)
+    existing_runner = False
+    collect_only = ("--co" in sys_argv or "--collect-only" in sys_argv)
+    all_scripts = (hasattr(sb_config, "all_scripts") and sb_config.all_scripts)
+    if (
+        (hasattr(sb_config, "is_behave") and sb_config.is_behave)
+        or (hasattr(sb_config, "is_pytest") and sb_config.is_pytest)
+        or (hasattr(sb_config, "is_nosetest") and sb_config.is_nosetest)
+    ):
+        existing_runner = True
+    if (
+        existing_runner
+        and not hasattr(sb_config, "_context_of_runner")
+    ):
+        if hasattr(sb_config, "is_pytest") and sb_config.is_pytest:
+            import pytest
+            msg = "Skipping `Driver()` script. (Use `python`, not `pytest`)"
+            if not collect_only and not all_scripts:
+                print("\n  *** %s" % msg)
+            if collect_only or not all_scripts:
+                pytest.skip(allow_module_level=True)
+        elif hasattr(sb_config, "is_nosetest") and sb_config.is_nosetest:
+            raise Exception(
+                "\n  A Driver() script was triggered by nosetest collection!"
+                '\n  (Prevent that by using: ``if __name__ == "__main__":``)'
+            )
+    elif existing_runner:
+        sb_config._context_of_runner = True
     browser_changes = 0
     browser_set = None
     browser_text = None
diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py
@@ -759,6 +759,16 @@ def pytest_addoption(parser):
         help="""This is used by the BaseCase class to tell apart
                 pytest runs from nosetest runs. (Automatic)""",
     )
+    parser.addoption(
+        "--all-scripts",
+        "--all_scripts",
+        action="store_true",
+        dest="all_scripts",
+        default=False,
+        help="""Use this to run `SB()`, `DriverContext()` and
+                `Driver()` scripts that are discovered during
+                the pytest collection phase.""",
+    )
     parser.addoption(
         "--time_limit",
         "--time-limit",
@@ -1525,6 +1535,7 @@ def pytest_configure(config):
         settings.ARCHIVE_EXISTING_DOWNLOADS = True
     if config.getoption("skip_js_waits"):
         settings.SKIP_JS_WAITS = True
+    sb_config.all_scripts = config.getoption("all_scripts")
     sb_config._time_limit = config.getoption("time_limit")
     sb_config.time_limit = config.getoption("time_limit")
     sb_config.slow_mode = config.getoption("slow_mode")
diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py
@@ -136,6 +136,8 @@ def SB(
     arg_join = " ".join(sys_argv)
     archive_logs = False
     existing_runner = False
+    collect_only = ("--co" in sys_argv or "--collect-only" in sys_argv)
+    all_scripts = (hasattr(sb_config, "all_scripts") and sb_config.all_scripts)
     do_log_folder_setup = False  # The first "test=True" run does it
     if (
         (hasattr(sb_config, "is_behave") and sb_config.is_behave)
@@ -146,21 +148,21 @@ def SB(
         test = False  # Already using a test runner. Skip extra test steps.
     elif test is None and "--test" in sys_argv:
         test = True
-    if (
-        existing_runner
-        and not hasattr(sb_config, "_context_of_runner")
-    ):
-        sb_config._context_of_runner = True
+    if existing_runner and not hasattr(sb_config, "_context_of_runner"):
         if hasattr(sb_config, "is_pytest") and sb_config.is_pytest:
-            print(
-                "\n  SB Manager script was triggered by pytest collection!"
-                '\n  (Prevent that by using: `if __name__ == "__main__":`)'
-            )
+            import pytest
+            msg = "Skipping `SB()` script. (Use `python`, not `pytest`)"
+            if not collect_only and not all_scripts:
+                print("\n  *** %s" % msg)
+            if collect_only or not all_scripts:
+                pytest.skip(allow_module_level=True)
         elif hasattr(sb_config, "is_nosetest") and sb_config.is_nosetest:
             raise Exception(
                 "\n  SB Manager script was triggered by nosetest collection!"
                 '\n  (Prevent that by using: ``if __name__ == "__main__":``)'
             )
+    elif existing_runner:
+        sb_config._context_of_runner = True
     if (
         not existing_runner
         and not hasattr(sb_config, "_has_older_context")
@@ -959,8 +961,6 @@ def SB(
         sb_config = sb_config_backup
         if test:
             sb_config._has_older_context = True
-        if existing_runner:
-            sb_config._context_of_runner = True
         if test_name:
             result = "passed"
             if test and not test_passed:

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`# seleniumbase package`
`2`		`-__version__ = "4.24.3"`
	`2`	`+__version__ = "4.24.4"`