Merge pull request #3452 from seleniumbase/download-browsers-and-more

Add support for downloading browsers, and more

Author: mdmintz

.gitignore

@@ -82,6 +82,20 @@ msedgedriver.exe
 operadriver.exe
 uc_driver.exe
 
+# Chrome for Testing folders
+chrome-mac-arm64
+chrome-mac-x64
+chrome-linux64
+chrome-win64
+chrome-win32
+
+# Chrome-Headless-Shell folders
+chrome-headless-shell-mac-arm64
+chrome-headless-shell-mac-x64
+chrome-headless-shell-linux64
+chrome-headless-shell-win64
+chrome-headless-shell-win32
+
 # msedgedriver requirements
 libc++.dylib
 

examples/proxy_test.py

@@ -13,18 +13,30 @@ def test_proxy(self):
         if not self.page_load_strategy == "none" and not self.undetectable:
             # This page takes too long to load otherwise
             self.get_new_driver(page_load_strategy="none")
+        self.open("https://api.ipify.org/")
+        ip_address = self.get_text("body")
         self.open("https://ipinfo.io/")
-        self.wait_for_non_empty_text("form input", timeout=20)
-        ip_address = self.get_text('#ip-string span[class*="primary"] span')
+        self.type('input[name="search"]', ip_address, timeout=20)
+        self.click("form button span")
+        self.sleep(2)
+        self.click_if_visible("span.cursor-pointer", timeout=4)
         print("\n\nMy IP Address = %s\n" % ip_address)
         print("Displaying Host Info:")
-        text = self.get_text("#widget-scrollable-container").split("asn:")[0]
+        text = self.get_text("#block-summary").split("Hosted domains")[0]
+        rows = text.split("\n")
+        data = []
+        for row in rows:
+            if row.strip() != "":
+                data.append(row.strip())
+        print("\n".join(data).replace('\n"', " "))
+        print("\nDisplaying Geolocation Info:")
+        text = self.get_text("#block-geolocation").split("Coordinates")[0]
         rows = text.split("\n")
         data = []
         for row in rows:
             if row.strip() != "":
                 data.append(row.strip())
         print("\n".join(data).replace('\n"', " "))
         if not self.headless:
-            print("\nThe browser will close automatically in 7 seconds...")
-            self.sleep(7)
+            print("\nThe browser will close automatically in 3 seconds...")
+            self.sleep(3)

examples/test_cdp_ad_blocking.py

@@ -24,6 +24,7 @@ def test_cdp_network_blocking(self):
             ]})
         self.execute_cdp_cmd('Network.enable', {})
         self.open('https://www.w3schools.com/jquery/default.asp')
+        self.ad_block()
         source = self.get_page_source()
         self.assert_true("doubleclick.net" not in source)
         self.assert_true("google-analytics.com" not in source)

examples/youtube_search_test.py

@@ -11,7 +11,7 @@ def test_youtube_autocomplete_results(self):
             self.skip("Unsupported mode for this test.")
         self.open("https://www.youtube.com/c/MichaelMintz")
         search_term = "seleniumbase"
-        search_selector = "input#search"
+        search_selector = 'input[name="search_query"]'
         results_selector = '[role="listbox"]'
         self.click_if_visible('button[aria-label="Close"]')
         self.double_click(search_selector)

help_docs/customizing_test_runs.md

@@ -337,6 +337,22 @@ pytest --headless -n8 --dashboard --html=report.html -v --rs --crumbs
 
 The above not only runs tests in parallel processes, but it also tells tests in the same process to share the same browser session, runs the tests in headless mode, displays the full name of each test on a separate line, creates a real-time dashboard of the test results, and creates a full report after all tests complete.
 
+🎛️ For extra speed, run your tests using `chrome-headless-shell`:
+
+First, get `chrome-headless-shell` if you don't already have it:
+
+```bash
+sbase get chs
+```
+
+Then, run scripts with `binary_location` / `bl` set to `"chs"`:
+
+```bash
+pytest --bl="chs" -n8 --dashboard --html=report.html -v --rs
+```
+
+That makes your tests run very quickly in headless mode.
+
 --------
 
 <h3><img src="https://seleniumbase.github.io/img/green_logo.png" title="SeleniumBase" width="32" /> The SeleniumBase Dashboard:</h3>
@@ -449,6 +465,27 @@ Note that different options could lead to the same result. (Eg. If you have the
 
 --------
 
+<h3><img src="https://seleniumbase.github.io/img/green_logo.png" title="SeleniumBase" width="32" /> Setting the binary location:</h3>
+
+🔵 By default, SeleniumBase uses the browser binary detected on the System PATH.
+
+🎛️ To change this default behavior, you can use:
+
+```bash
+pytest --binary-location=PATH
+```
+
+The `PATH` in `--binary-location=PATH` / `--bl=PATH` can be:
+* A relative or exact path to the browser binary.
+* `"cft"` as a special option for `Chrome for Testing`.
+* `"chs"` as a special option for `Chrome-Headless-Shell`.
+
+Before using the `"cft"` / `"chs"` options, call `sbase get cft` / `sbase get chs` in order to download the specified binaries into the `seleniumbase/drivers` folder. The default version is the latest stable version on https://googlechromelabs.github.io/chrome-for-testing/. You can change that by specifying the arg as a parameter. (Eg. `sbase get cft 131`, `sbase get chs 132`, etc.)
+
+With the `SB()` and `Driver()` formats, the binary location is set via the `binary_location` parameter.
+
+--------
+
 <h3><img src="https://seleniumbase.github.io/img/green_logo.png" title="SeleniumBase" width="32" /> Customizing default settings:</h3>
 
 🎛️ An easy way to override [seleniumbase/config/settings.py](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/config/settings.py) is by using a custom settings file.

help_docs/webdriver_installation.md

@@ -4,7 +4,7 @@
 
 To run web automation, you need webdrivers for each browser you plan on using.  With SeleniumBase, drivers are downloaded automatically (as needed) into the SeleniumBase `drivers/` folder.
 
-You can also download drivers manually with these commands:
+🎛️ You can also download drivers manually with these commands:
 
 ```bash
 seleniumbase get chromedriver
@@ -16,7 +16,7 @@ After running the commands above, web drivers will get downloaded into the `sele
 
 If the necessary driver is not found in this location while running tests, SeleniumBase will instead look for the driver on the System PATH. If the necessary driver is not on the System PATH either, SeleniumBase will automatically attempt to download the required driver.
 
-* You can also download specific versions of drivers. Examples:
+🎛️ You can also download specific versions of drivers. Examples:
 
 ```bash
 sbase get chromedriver 114
@@ -30,7 +30,7 @@ sbase get chromedriver mlatest  # Milestone latest version for detected browser
 sbase get edgedriver 115.0.1901.183
 ```
 
-(NOTE: ``sbase`` is a shortcut for ``seleniumbase``)
+(NOTE: `sbase` is a shortcut for `seleniumbase`)
 
 --------
 
@@ -48,15 +48,15 @@ Here's where you can go to manually get web drivers from the source:
 
 **macOS shortcuts**:
 
-* You can also install drivers by using ``brew`` (aka ``homebrew``):
+🎛️ You can also install drivers by using ``brew`` (aka ``homebrew``):
 
 ```bash
 brew install --cask chromedriver
 
 brew install geckodriver
 ```
 
-You can also upgrade existing webdrivers:
+🎛️ You can also upgrade existing webdrivers:
 
 ```bash
 brew upgrade --cask chromedriver
@@ -66,7 +66,7 @@ brew upgrade geckodriver
 
 **Linux shortcuts**:
 
-If you still need drivers, these scripts download ``chromedriver`` and ``geckodriver`` to a Linux machine:
+🎛️ If you still need drivers, these scripts download `chromedriver` and `geckodriver` to a Linux machine:
 
 ```bash
 wget https://chromedriver.storage.googleapis.com/114.0.5735.90/chromedriver_linux64.zip
@@ -76,12 +76,30 @@ chmod +x /usr/local/bin/chromedriver
 ```
 
 ```bash
-wget https://github.com/mozilla/geckodriver/releases/download/v0.34.0/geckodriver-v0.34.0-linux64.tar.gz
-tar xvfz geckodriver-v0.34.0-linux64.tar.gz
+wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz
+tar xvfz geckodriver-v0.35.0-linux64.tar.gz
 mv geckodriver /usr/local/bin/
 chmod +x /usr/local/bin/geckodriver
 ```
 
 To verify that web drivers are working, **[follow these instructions](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/verify_webdriver.md)**.
 
+--------
+
+**Browser Binaries**:
+
+🎛️ Use the `sbase get` command to download the `Chrome for Testing` and `Chrome-Headless-Shell` browser binaries. Example:
+
+```bash
+sbase get cft  # (For `Chrome for Testing`)
+sbase get chs  # (For `Chrome-Headless-Shell`)
+```
+
+Those commands download those binaries into the `seleniumbase/drivers` folder.
+To use the binaries from there in SeleniumBase scripts, set the `binary_location` to `cft` or `chs`.
+
+(Source: https://googlechromelabs.github.io/chrome-for-testing/)
+
+--------
+
 [<img src="https://seleniumbase.github.io/cdn/img/sb_logo_b.png" title="SeleniumBase" width="280">](https://github.com/seleniumbase/SeleniumBase)

mkdocs_build/requirements.txt

@@ -6,17 +6,13 @@ pymdown-extensions>=10.14.1
 pipdeptree>=2.24.0
 python-dateutil>=2.8.2
 Markdown==3.7
-markdown2==2.5.2
 click==8.1.8
 ghp-import==2.1.0
 watchdog==6.0.0
 cairocffi==1.7.1
 pathspec==0.12.1
 Babel==2.16.0
 paginate==0.5.7
-lxml==5.3.0
-pyquery==2.0.1
-readtime==3.0.0
 mkdocs==1.6.1
 mkdocs-material==9.5.50
 mkdocs-exclude-search==0.6.6

requirements.txt

@@ -43,7 +43,7 @@ trio-websocket==0.11.1
 wsproto==1.2.0
 websocket-client==1.8.0
 selenium==4.27.1;python_version<"3.9"
-selenium==4.28.0;python_version>="3.9"
+selenium==4.28.1;python_version>="3.9"
 cssselect==1.2.0
 sortedcontainers==2.4.0
 execnet==2.1.1

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.34.1"
+__version__ = "4.34.2"

seleniumbase/behave/behave_sb.py

@@ -482,8 +482,8 @@ def get_configured_sb(context):
                 extension_dir = sb.extension_dir  # revert to default
             sb.extension_dir = extension_dir
             continue
-        # Handle: -D binary-location=PATH / binary_location=PATH
-        if low_key in ["binary-location", "binary_location"]:
+        # Handle: -D binary-location=PATH / binary_location=PATH / bl=PATH
+        if low_key in ["binary-location", "binary_location", "bl"]:
             binary_location = userdata[key]
             if binary_location == "true":
                 binary_location = sb.binary_location  # revert to default
@@ -884,6 +884,14 @@ def get_configured_sb(context):
         sb.headless = True  # Firefox has regular headless
     elif sb.browser not in ["chrome", "edge"]:
         sb.headless2 = False  # Only for Chromium browsers
+    if (
+        sb.binary_location
+        and sb.binary_location.lower() == "chs"
+        and sb.browser == "chrome"
+    ):
+        sb.headless = True
+        sb.headless1 = False
+        sb.headless2 = False
     # Recorder Mode only supports Chromium browsers.
     if sb.recorder_ext and (sb.browser not in ["chrome", "edge"]):
         raise Exception(

seleniumbase/console_scripts/ReadMe.md

@@ -68,10 +68,12 @@ sbase get chromedriver 114.0.5735.90
 sbase get chromedriver stable
 sbase get chromedriver beta
 sbase get chromedriver -p
+sbase get cft 131
+sbase get chs
 ```
 
-(Drivers:  ``chromedriver``, ``geckodriver``, ``edgedriver``,
-           ``iedriver``, ``uc_driver``)
+(Drivers:  ``chromedriver``, ``cft``, ``uc_driver``,
+           ``edgedriver``, ``chs``, ``geckodriver``)
 
 (Options:  A specific driver version or major version integer.
            If not set, the driver version matches the browser.

seleniumbase/console_scripts/run.py

@@ -139,8 +139,8 @@ def show_install_usage():
     print("     OR: seleniumbase get [DRIVER_NAME] [OPTIONS]")
     print("     OR:    sbase install [DRIVER_NAME] [OPTIONS]")
     print("     OR:        sbase get [DRIVER_NAME] [OPTIONS]")
-    print("         (Drivers: chromedriver, geckodriver,")
-    print("                   edgedriver, iedriver, uc_driver)")
+    print("         (Drivers: chromedriver, cft, uc_driver,")
+    print("                   edgedriver, chs, geckodriver)")
     print("  Options:")
     print("     VERSION    Specify the version to download.")
     print("                Tries to detect the needed version.")
@@ -157,11 +157,15 @@ def show_install_usage():
     print("     sbase get chromedriver stable")
     print("     sbase get chromedriver beta")
     print("     sbase get chromedriver -p")
+    print("     sbase get cft 131")
+    print("     sbase get chs")
     print("  Output:")
     print("     Downloads the webdriver to seleniumbase/drivers/")
     print("     (chromedriver is required for Chrome automation)")
     print("     (geckodriver is required for Firefox automation)")
     print("     (edgedriver is required for MS__Edge automation)")
+    print("     (cft is for the `Chrome for Testing` binary exe)")
+    print("     (chs is for the `Chrome-Headless-Shell` binary.)")
     print("")