diff --git a/.agents/skills/scrapingbee-cli/reference/amazon/product.md b/.agents/skills/scrapingbee-cli/reference/amazon/product.md index 002512e..0f67db0 100644 --- a/.agents/skills/scrapingbee-cli/reference/amazon/product.md +++ b/.agents/skills/scrapingbee-cli/reference/amazon/product.md @@ -21,6 +21,7 @@ scrapingbee amazon-product --output-file product.json B0DPDRNSXV --domain com | `--language` | string | e.g. en_US, es_US, fr_FR. | | `--currency` | string | USD, EUR, GBP, etc. | | `--add-html` | true/false | Include full HTML. | +| `--autoselect-variant` | true/false | Auto-select the default/most-popular variant (undocumented API param, verified accepted). | | `--light-request` | true/false | Light request. | | `--screenshot` | true/false | Take screenshot. | | `--tag` | string | Optional label included in API response headers. | diff --git a/.agents/skills/scrapingbee-cli/reference/google/overview.md b/.agents/skills/scrapingbee-cli/reference/google/overview.md index b371703..87d4959 100644 --- a/.agents/skills/scrapingbee-cli/reference/google/overview.md +++ b/.agents/skills/scrapingbee-cli/reference/google/overview.md @@ -19,6 +19,7 @@ scrapingbee google --output-file serp.json "pizza new york" --country-code us | `--device` | string | `desktop` or `mobile`. | | `--page` | int | Page number (default 1). | | `--pages` | int | Consecutive pages to fetch starting at `--page` (default 1, max 10; 3 or fewer recommended). Combined into one response; cost is per fetched page. | +| `--nb-results` | int | Requested results per page (undocumented API param, verified accepted; Google may return more or fewer). | | `--language` | string | Language code (e.g. en, fr, de). | | `--date-range` | string | `past-hour`, `past-day`, `past-week`, `past-month`, `past-year`. Restrict results by recency. | | `--nfpr` | true/false | Disable autocorrection. | diff --git a/.agents/skills/scrapingbee-cli/reference/scrape/options.md b/.agents/skills/scrapingbee-cli/reference/scrape/options.md index 98f2800..37ee290 100644 --- a/.agents/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.agents/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (15 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.github/skills/scrapingbee-cli/reference/amazon/product.md b/.github/skills/scrapingbee-cli/reference/amazon/product.md index 002512e..0f67db0 100644 --- a/.github/skills/scrapingbee-cli/reference/amazon/product.md +++ b/.github/skills/scrapingbee-cli/reference/amazon/product.md @@ -21,6 +21,7 @@ scrapingbee amazon-product --output-file product.json B0DPDRNSXV --domain com | `--language` | string | e.g. en_US, es_US, fr_FR. | | `--currency` | string | USD, EUR, GBP, etc. | | `--add-html` | true/false | Include full HTML. | +| `--autoselect-variant` | true/false | Auto-select the default/most-popular variant (undocumented API param, verified accepted). | | `--light-request` | true/false | Light request. | | `--screenshot` | true/false | Take screenshot. | | `--tag` | string | Optional label included in API response headers. | diff --git a/.github/skills/scrapingbee-cli/reference/google/overview.md b/.github/skills/scrapingbee-cli/reference/google/overview.md index b371703..87d4959 100644 --- a/.github/skills/scrapingbee-cli/reference/google/overview.md +++ b/.github/skills/scrapingbee-cli/reference/google/overview.md @@ -19,6 +19,7 @@ scrapingbee google --output-file serp.json "pizza new york" --country-code us | `--device` | string | `desktop` or `mobile`. | | `--page` | int | Page number (default 1). | | `--pages` | int | Consecutive pages to fetch starting at `--page` (default 1, max 10; 3 or fewer recommended). Combined into one response; cost is per fetched page. | +| `--nb-results` | int | Requested results per page (undocumented API param, verified accepted; Google may return more or fewer). | | `--language` | string | Language code (e.g. en, fr, de). | | `--date-range` | string | `past-hour`, `past-day`, `past-week`, `past-month`, `past-year`. Restrict results by recency. | | `--nfpr` | true/false | Disable autocorrection. | diff --git a/.github/skills/scrapingbee-cli/reference/scrape/options.md b/.github/skills/scrapingbee-cli/reference/scrape/options.md index 98f2800..37ee290 100644 --- a/.github/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.github/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (15 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.kiro/skills/scrapingbee-cli/reference/amazon/product.md b/.kiro/skills/scrapingbee-cli/reference/amazon/product.md index 002512e..0f67db0 100644 --- a/.kiro/skills/scrapingbee-cli/reference/amazon/product.md +++ b/.kiro/skills/scrapingbee-cli/reference/amazon/product.md @@ -21,6 +21,7 @@ scrapingbee amazon-product --output-file product.json B0DPDRNSXV --domain com | `--language` | string | e.g. en_US, es_US, fr_FR. | | `--currency` | string | USD, EUR, GBP, etc. | | `--add-html` | true/false | Include full HTML. | +| `--autoselect-variant` | true/false | Auto-select the default/most-popular variant (undocumented API param, verified accepted). | | `--light-request` | true/false | Light request. | | `--screenshot` | true/false | Take screenshot. | | `--tag` | string | Optional label included in API response headers. | diff --git a/.kiro/skills/scrapingbee-cli/reference/google/overview.md b/.kiro/skills/scrapingbee-cli/reference/google/overview.md index b371703..87d4959 100644 --- a/.kiro/skills/scrapingbee-cli/reference/google/overview.md +++ b/.kiro/skills/scrapingbee-cli/reference/google/overview.md @@ -19,6 +19,7 @@ scrapingbee google --output-file serp.json "pizza new york" --country-code us | `--device` | string | `desktop` or `mobile`. | | `--page` | int | Page number (default 1). | | `--pages` | int | Consecutive pages to fetch starting at `--page` (default 1, max 10; 3 or fewer recommended). Combined into one response; cost is per fetched page. | +| `--nb-results` | int | Requested results per page (undocumented API param, verified accepted; Google may return more or fewer). | | `--language` | string | Language code (e.g. en, fr, de). | | `--date-range` | string | `past-hour`, `past-day`, `past-week`, `past-month`, `past-year`. Restrict results by recency. | | `--nfpr` | true/false | Disable autocorrection. | diff --git a/.kiro/skills/scrapingbee-cli/reference/scrape/options.md b/.kiro/skills/scrapingbee-cli/reference/scrape/options.md index 98f2800..37ee290 100644 --- a/.kiro/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.kiro/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (15 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/.opencode/skills/scrapingbee-cli/reference/amazon/product.md b/.opencode/skills/scrapingbee-cli/reference/amazon/product.md index 002512e..0f67db0 100644 --- a/.opencode/skills/scrapingbee-cli/reference/amazon/product.md +++ b/.opencode/skills/scrapingbee-cli/reference/amazon/product.md @@ -21,6 +21,7 @@ scrapingbee amazon-product --output-file product.json B0DPDRNSXV --domain com | `--language` | string | e.g. en_US, es_US, fr_FR. | | `--currency` | string | USD, EUR, GBP, etc. | | `--add-html` | true/false | Include full HTML. | +| `--autoselect-variant` | true/false | Auto-select the default/most-popular variant (undocumented API param, verified accepted). | | `--light-request` | true/false | Light request. | | `--screenshot` | true/false | Take screenshot. | | `--tag` | string | Optional label included in API response headers. | diff --git a/.opencode/skills/scrapingbee-cli/reference/google/overview.md b/.opencode/skills/scrapingbee-cli/reference/google/overview.md index b371703..87d4959 100644 --- a/.opencode/skills/scrapingbee-cli/reference/google/overview.md +++ b/.opencode/skills/scrapingbee-cli/reference/google/overview.md @@ -19,6 +19,7 @@ scrapingbee google --output-file serp.json "pizza new york" --country-code us | `--device` | string | `desktop` or `mobile`. | | `--page` | int | Page number (default 1). | | `--pages` | int | Consecutive pages to fetch starting at `--page` (default 1, max 10; 3 or fewer recommended). Combined into one response; cost is per fetched page. | +| `--nb-results` | int | Requested results per page (undocumented API param, verified accepted; Google may return more or fewer). | | `--language` | string | Language code (e.g. en, fr, de). | | `--date-range` | string | `past-hour`, `past-day`, `past-week`, `past-month`, `past-year`. Restrict results by recency. | | `--nfpr` | true/false | Disable autocorrection. | diff --git a/.opencode/skills/scrapingbee-cli/reference/scrape/options.md b/.opencode/skills/scrapingbee-cli/reference/scrape/options.md index 98f2800..37ee290 100644 --- a/.opencode/skills/scrapingbee-cli/reference/scrape/options.md +++ b/.opencode/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (15 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 0897119..3205d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.6.0] - TBD +## [1.6.0] - 2026-08-24 ### Added @@ -15,6 +15,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`youtube-subtitles` command** — fetch video captions/transcripts from the YouTube Subtitles API (5 credits per request). Accepts a video ID or full YouTube URL, `--language` (ISO code) and `--subtitle-origin` (`auto-generated` / `uploader-provided`), and supports batch via `--input-file` like the other YouTube commands. - **`--pages` on `google`** — fetch up to 10 consecutive result pages starting at `--page` in a single combined response (3 or fewer recommended; cost is per fetched page). - **`--search-type ads` on `google`** — classic-result structure optimized for paid-ad visibility. +- **`--nb-results` on `google`** — requested number of results per page. Undocumented API parameter, verified accepted by the API (Google may return more or fewer results than requested). +- **`--autoselect-variant` on `amazon-product`** — auto-select the default/most-popular product variant, matching the existing `amazon-search` flag. Undocumented API parameter, verified accepted by the API. + +### Changed + +- **Header-based authorization** — all API requests now authenticate via the `Authorization: Bearer` header instead of the deprecated `api_key` query parameter, so the key no longer appears in request URLs (or anything that logs them). `crawl` is the one exception: its Scrapy middleware (`scrapy-scrapingbee`) still builds `api_key` URLs and will migrate separately. ## [1.5.1] - 2026-07-20 diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/amazon/product.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/amazon/product.md index 002512e..0f67db0 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/amazon/product.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/amazon/product.md @@ -21,6 +21,7 @@ scrapingbee amazon-product --output-file product.json B0DPDRNSXV --domain com | `--language` | string | e.g. en_US, es_US, fr_FR. | | `--currency` | string | USD, EUR, GBP, etc. | | `--add-html` | true/false | Include full HTML. | +| `--autoselect-variant` | true/false | Auto-select the default/most-popular variant (undocumented API param, verified accepted). | | `--light-request` | true/false | Light request. | | `--screenshot` | true/false | Take screenshot. | | `--tag` | string | Optional label included in API response headers. | diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/google/overview.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/google/overview.md index b371703..87d4959 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/google/overview.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/google/overview.md @@ -19,6 +19,7 @@ scrapingbee google --output-file serp.json "pizza new york" --country-code us | `--device` | string | `desktop` or `mobile`. | | `--page` | int | Page number (default 1). | | `--pages` | int | Consecutive pages to fetch starting at `--page` (default 1, max 10; 3 or fewer recommended). Combined into one response; cost is per fetched page. | +| `--nb-results` | int | Requested results per page (undocumented API param, verified accepted; Google may return more or fewer). | | `--language` | string | Language code (e.g. en, fr, de). | | `--date-range` | string | `past-hour`, `past-day`, `past-week`, `past-month`, `past-year`. Restrict results by recency. | | `--nfpr` | true/false | Disable autocorrection. | diff --git a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md index 98f2800..37ee290 100644 --- a/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md +++ b/plugins/scrapingbee-cli/skills/scrapingbee-cli/reference/scrape/options.md @@ -71,7 +71,7 @@ Blocked? See [reference/proxy/strategies.md](reference/proxy/strategies.md). |-----------|------|-------------| | `--device` | desktop \| mobile | Device type (CLI validates). | | `--timeout` | int | Timeout ms (1000–140000). Scrape job timeout on ScrapingBee. The CLI sets the HTTP client (aiohttp) timeout to this value in seconds plus 30 s (for send/receive) so the client does not give up before the API responds. | -| `--custom-google` / `--transparent-status-code` | — | Google (15 credits), target status. | +| `--custom-google` / `--transparent-status-code` | — | Google (20 credits), target status. | | `--tag` | string | Optional label included in API response headers. | | `--mode` | auto | Auto-Mode: API picks the cheapest config that succeeds; charged only for the winning config. GET only. See [Auto-Mode](#auto-mode). | | `--max-cost` | int | Cap credits a request may cost (≥ 1). Requires `--mode auto`; omit = uncapped. | diff --git a/src/scrapingbee_cli/client.py b/src/scrapingbee_cli/client.py index 5233454..c6ee69b 100644 --- a/src/scrapingbee_cli/client.py +++ b/src/scrapingbee_cli/client.py @@ -43,10 +43,13 @@ async def __aenter__(self) -> Client: else aiohttp.TCPConnector(ssl=ssl_context) ) timeout = aiohttp.ClientTimeout(total=self.timeout) + # Header-based auth: the api_key query parameter is deprecated for + # new integrations, so authenticate every request via the session. + session_headers = user_agent_headers() | {"Authorization": f"Bearer {self.api_key}"} self._session = aiohttp.ClientSession( connector=connector, timeout=timeout, - headers=user_agent_headers(), + headers=session_headers, ) return self @@ -67,7 +70,6 @@ async def _get( headers: dict[str, str] | None = None, ) -> tuple[bytes, dict, int]: params = _clean_params(params) - params.setdefault("api_key", self.api_key) url = f"{self.base_url}{path}" if path else self.base_url session = self._ensure_session() req_kwargs: dict[str, Any] = {"params": params} @@ -114,7 +116,6 @@ async def _request( headers: dict[str, str] | None = None, ) -> tuple[bytes, dict, int]: params = _clean_params(params) - params.setdefault("api_key", self.api_key) url = f"{self.base_url}{path}" if path else self.base_url session = self._ensure_session() req_kwargs: dict[str, Any] = {"params": params} @@ -243,7 +244,6 @@ async def scrape( body_out, out_headers, status = await self._get("", params, headers=req_headers) else: params_clean = _clean_params(params) - params_clean["api_key"] = self.api_key # ScrapingBee API expects POST to it as application/x-www-form-urlencoded content_type = "application/x-www-form-urlencoded; charset=utf-8" # Don't send user's Content-Type to ScrapingBee; forward via params if needed @@ -279,7 +279,7 @@ async def usage( ) -> tuple[bytes, dict, int]: return await self._get_with_retry( "/usage", - {"api_key": self.api_key}, + {}, retries=retries, backoff=backoff, ) @@ -292,6 +292,7 @@ async def google_search( device: str | None = None, page: int | None = None, pages: int | None = None, + nb_results: int | None = None, language: str | None = None, nfpr: bool | None = None, extra_params: str | None = None, @@ -315,6 +316,7 @@ async def google_search( "device": device, "page": page if page is not None else None, "pages": pages if pages is not None else None, + "nb_results": nb_results if nb_results is not None else None, "language": language, "nfpr": self._bool(nfpr), "extra_params": extra_params, @@ -369,6 +371,7 @@ async def amazon_product( zip_code: str | None = None, language: str | None = None, currency: str | None = None, + autoselect_variant: bool | None = None, add_html: bool | None = None, light_request: bool | None = None, screenshot: bool | None = None, @@ -384,6 +387,7 @@ async def amazon_product( "zip_code": zip_code, "language": language, "currency": currency, + "autoselect_variant": self._bool(autoselect_variant), "add_html": self._bool(add_html), "light_request": self._bool(light_request), "screenshot": self._bool(screenshot), diff --git a/src/scrapingbee_cli/commands/amazon.py b/src/scrapingbee_cli/commands/amazon.py index 00a5806..0835863 100644 --- a/src/scrapingbee_cli/commands/amazon.py +++ b/src/scrapingbee_cli/commands/amazon.py @@ -62,6 +62,15 @@ ) @optgroup.option("--currency", type=str, default=None, help="Currency code (e.g. USD, EUR, GBP).") @optgroup.group("Output", help="Response format options") +@optgroup.option( + "--autoselect-variant", + type=BOOL_STR, + default=None, + help=( + "Auto-select the default/most-popular product variant (true/false). " + "Undocumented API parameter — verified accepted by the API." + ), +) @optgroup.option( "--add-html", type=BOOL_STR, default=None, help="Include full HTML in response (true/false)." ) @@ -86,6 +95,7 @@ def amazon_product_cmd( zip_code: str | None, language: str | None, currency: str | None, + autoselect_variant: str | None, add_html: str | None, light_request: str | None, screenshot: str | None, @@ -135,6 +145,7 @@ async def api_call(client, a): zip_code=zip_code, language=language, currency=currency, + autoselect_variant=parse_bool(autoselect_variant), add_html=parse_bool(add_html), light_request=parse_bool(light_request), screenshot=parse_bool(screenshot), @@ -178,6 +189,7 @@ async def _single() -> None: zip_code=zip_code, language=language, currency=currency, + autoselect_variant=parse_bool(autoselect_variant), add_html=parse_bool(add_html), light_request=parse_bool(light_request), screenshot=parse_bool(screenshot), diff --git a/src/scrapingbee_cli/commands/google.py b/src/scrapingbee_cli/commands/google.py index 508d499..2ad1dee 100644 --- a/src/scrapingbee_cli/commands/google.py +++ b/src/scrapingbee_cli/commands/google.py @@ -93,6 +93,15 @@ def _warn_empty_organic(data: bytes, search_type: str | None) -> None: "3 or fewer recommended). Results are combined into one response." ), ) +@optgroup.option( + "--nb-results", + type=int, + default=None, + help=( + "Requested number of results per page (undocumented API parameter — " + "verified accepted by the API; Google may return more or fewer)." + ), +) @optgroup.option( "--language", type=str, @@ -178,6 +187,7 @@ def google_cmd( device: str | None, page: int | None, pages: int | None, + nb_results: int | None, language: str | None, nfpr: str | None, extra_params: str | None, @@ -206,6 +216,7 @@ def google_cmd( raise SystemExit(1) _validate_page(page) _validate_pages(pages) + _validate_page(nb_results, name="nb-results") _validate_price_range(min_price, max_price) _validate_geolocation(latitude, longitude, radius) @@ -239,6 +250,7 @@ async def api_call(client, q): device=device, page=page, pages=pages, + nb_results=nb_results, language=language, nfpr=parse_bool(nfpr), extra_params=extra_params, @@ -290,6 +302,7 @@ async def _single() -> None: device=device, page=page, pages=pages, + nb_results=nb_results, language=language, nfpr=parse_bool(nfpr), extra_params=extra_params, diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index dffa99d..171312e 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -260,6 +260,20 @@ def test_google_pages_option(self): assert code == 0 assert "--pages" in out + def test_google_nb_results_option(self): + from tests.conftest import cli_run + + code, out, _ = cli_run(["google", "--help"]) + assert code == 0 + assert "--nb-results" in out + + def test_amazon_product_autoselect_variant_option(self): + from tests.conftest import cli_run + + code, out, _ = cli_run(["amazon-product", "--help"]) + assert code == 0 + assert "--autoselect-variant" in out + class TestExtractFieldValues: """Tests for _extract_field_values().""" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index c8966bf..2aac0fc 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -378,6 +378,134 @@ async def fake_get(path, params, headers=None): asyncio.run(run()) +class TestHeaderAuth: + """The client authenticates via Authorization: Bearer, not the deprecated api_key param.""" + + def test_session_has_bearer_authorization(self): + async def run(): + async with Client("fake-key") as client: + assert client._ensure_session().headers.get("Authorization") == "Bearer fake-key" + + asyncio.run(run()) + + def test_get_sends_no_api_key_param(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.scrape("https://example.com", retries=0) + assert "api_key" not in captured["params"] + + asyncio.run(run()) + + def test_post_sends_no_api_key_param(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_request( + method, path, params, data=None, content_type=None, headers=None + ): + captured["params"] = dict(params) + return (b"{}", {}, 200) + + with patch.object(client, "_request", new=AsyncMock(side_effect=fake_request)): + await client.scrape("https://example.com", method="post", body="x=1", retries=0) + assert "api_key" not in captured["params"] + + asyncio.run(run()) + + def test_usage_sends_no_api_key_param(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.usage(retries=0) + assert "api_key" not in captured["params"] + + asyncio.run(run()) + + +class TestGoogleNbResults: + """Tests that google_search forwards nb_results only when set.""" + + def test_nb_results_sent_when_set(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.google_search("coffee", nb_results=3, retries=0) + assert captured["params"].get("nb_results") == 3 + + asyncio.run(run()) + + def test_nb_results_omitted_when_unset(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.google_search("coffee", retries=0) + assert "nb_results" not in captured["params"] + + asyncio.run(run()) + + +class TestAmazonProductAutoselectVariant: + """Tests that amazon_product forwards autoselect_variant only when set.""" + + def test_sent_when_set(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["path"] = path + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.amazon_product("B000000000", autoselect_variant=True, retries=0) + assert captured["path"] == "/amazon/product" + assert captured["params"].get("autoselect_variant") == "true" + + asyncio.run(run()) + + def test_omitted_when_unset(self): + async def run(): + client = Client("fake-key") + captured: dict = {} + + async def fake_get(path, params, headers=None): + captured["params"] = _clean_params(params) + return (b"{}", {}, 200) + + with patch.object(client, "_get", new=AsyncMock(side_effect=fake_get)): + await client.amazon_product("B000000000", retries=0) + assert "autoselect_variant" not in captured["params"] + + asyncio.run(run()) + + class TestGoogleDateRange: """Tests that google_search forwards date_range only when set.""" diff --git a/tests/unit/test_repl_pty.py b/tests/unit/test_repl_pty.py index 533cf0d..7e54448 100644 --- a/tests/unit/test_repl_pty.py +++ b/tests/unit/test_repl_pty.py @@ -657,19 +657,20 @@ def test_session_default_skip_warning_on_screen(tmp_path): stream, lambda s: "premium-proxy" in _text(s) and "true" in _text(s), ), ":set did not apply premium-proxy=true" - child.send("google --help\r") - # Match on the raw stream, not the screen: the warning is one line - # printed before the full --help output, which scrolls it out of the - # 32-row viewport — a screen predicate only wins the race when an - # intermediate frame happens to be captured. - matched, _ = _pump_until_raw( + # ``usage --help`` keeps the output short so the warning stays on + # screen; the transient pump checks every intermediate screen state, + # not just the post-read one. A long help (e.g. ``google --help``) + # is unreliable here: the differential renderer may repaint straight + # to the final frame without ever emitting the scrolled-off warning + # into the PTY stream. + child.send("usage --help\r") + assert _pump_until_transient( child, screen, stream, - lambda t: _has_session_default_skip_warning(t, "google", "premium-proxy"), + lambda s: _has_session_default_skip_warning(_text(s), "usage", "premium-proxy"), timeout=20.0, - ) - assert matched, "skip warning for premium-proxy on google not shown" + ), f"skip warning for premium-proxy on usage not shown; screen:\n{_text(screen)}" finally: child.close(force=True)