From ac75e9fc6fbb408b1009e93cb508f2a653026465 Mon Sep 17 00:00:00 2001 From: Adarsh Divakaran Date: Mon, 16 Nov 2026 00:47:19 +0530 Subject: [PATCH 1/4] feat: add AsyncClient via optional async extra Add serpapi.AsyncClient and AsyncSerpResults, backed by httpx2 through the optional serpapi[async] extra on Python 3.11+. The synchronous Client stays on requests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CONTRIBUTING.md | 8 +- README.md | 44 +- docs/conf.py | 2 +- docs/index.md | 4 +- docs/reference.md | 4 + docs/user_guide/async-client.md | 347 +++++++++++++ docs/user_guide/async-search-archive.md | 2 +- docs/user_guide/client-usage.md | 6 +- docs/user_guide/errors-and-timeouts.md | 2 +- docs/user_guide/getting-started.md | 3 +- docs/user_guide/index.md | 1 + docs/user_guide/request-options.md | 4 +- docs/user_guide/threading.md | 2 +- pyproject.toml | 3 +- serpapi/core.py | 212 +++++++- serpapi/exceptions.py | 7 +- serpapi/http.py | 375 ++++++++++++++ serpapi/models.py | 42 +- serpapi/utils.py | 14 + tests/docs_example_support.py | 30 +- tests/test_async_client.py | 635 ++++++++++++++++++++++++ tests/test_docs_example_runner.py | 38 ++ tests/test_exceptions.py | 15 + 23 files changed, 1774 insertions(+), 26 deletions(-) create mode 100644 docs/user_guide/async-client.md create mode 100644 tests/test_async_client.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74afa96..b6128e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,14 +31,16 @@ The commands below use `python` from that environment. ### Checks that need no API key -These tests cover response parsing, image uploads and extraction with simulated responses, timeouts, exceptions, documentation syntax, and the example runner: +These tests cover response parsing, image uploads and extraction with simulated responses, timeouts, exceptions, the async client, documentation syntax, and the example runner: ```sh -python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_extract.py tests/test_timeout.py tests/test_exceptions.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q +python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_extract.py tests/test_timeout.py tests/test_exceptions.py tests/test_async_client.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q ``` The file list limits this command to tests that make no external requests. Applying `-k 'not live'` to the entire suite is not enough to run offline: some SDK integration tests do not have `live` in their names. +The async client tests need Python 3.11 or newer, where the `test` extra installs httpx2. On older versions, pytest skips them. + ### SDK integration tests Set `API_KEY` in your shell or editor's run configuration before running integration tests. Documentation examples also accept `SERPAPI_KEY`, but the shared SDK fixtures require `API_KEY`. If you already have `SERPAPI_KEY` set, copy it to `API_KEY` in the same shell: @@ -67,7 +69,7 @@ Both commands make real SerpApi requests. The full suite can use more searches t The tests discover Python code blocks in the README and all Markdown pages under `docs/`. They run each page as a separate script against SerpApi, including the multiprocessing example's worker processes. The Lens upload examples use the repository's PNG icon as `image.png`. Each page has a five-minute limit, and HTTP errors or JSON responses containing an `error` fail the check even if the example catches the exception. -With `SERPAPI_KEY` or `API_KEY` set in your environment, run: +With `SERPAPI_KEY` or `API_KEY` set in your environment, run the following command with Python 3.11 or newer, because the README and the async client guide use `serpapi.AsyncClient`: ```sh python -m pytest tests/test_docs_examples.py --require-docs-key -q diff --git a/README.md b/README.md index aa061f2..855fe7f 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ To install the `serpapi` package, simply run the following command inside your P pip install serpapi ``` +To use `serpapi.AsyncClient` with `asyncio`, install the `async` extra instead. It requires Python 3.11 or later: + +```bash +pip install "serpapi[async]" +``` + +Without this extra, creating an `AsyncClient` raises an `ImportError` that includes the installation command. The synchronous `Client` works without it. + ## Quickstart [Create a SerpApi account](https://serpapi.com/users/sign_up?plan=free) to get your API key, then store it in an environment variable: @@ -82,6 +90,7 @@ A successful HTTP response can still contain an `error` field, such as when a se ## Features - **Asynchronous searches** for submitting jobs and retrieving completed results from the Search Archive API. +- **asyncio support** with `serpapi.AsyncClient` for running searches at the same time with `await`. - **Persistent connections and connection pooling** for reusing HTTP connections across searches. - Search results as a dict-like **SerpResults** object by default, token-efficient **Markdown** with `"output": "md"`, or raw **search-engine HTML** with `"output": "html"`. - SDK methods for the [Image API](https://serpapi.com/image-api), [Location API](https://serpapi.com/locations-api), [Search Archive API](https://serpapi.com/search-archive-api), and [Account API](https://serpapi.com/account-api). @@ -151,7 +160,7 @@ Learn more about [SerpApi Markdown Output](https://serpapi.com/markdown-output). Searches wait for their results by default. Use `{"async": True}` to submit a search job on SerpApi and retrieve its results later from the Search Archive API. -SerpApi's `async` parameter is separate from Python `asyncio`. The SDK methods are synchronous: each call blocks until its HTTP response arrives and cannot be used with `await`. +SerpApi's `async` parameter is separate from Python `asyncio`. `serpapi.Client` methods are synchronous: each call blocks until its HTTP response arrives. To use `await`, see [Async client](#async-client). > **Note:** `async` is a reserved keyword in Python 3.7+, so pass it inside the parameters dictionary (`{"async": True}`) rather than as a keyword argument. @@ -172,6 +181,39 @@ print(results["search_metadata"]["status"]) If the search is still processing, retrieve it again later. See the [async search guide](https://serpapi-python.readthedocs.io/en/latest/user_guide/async-search-archive.html) for polling with HTTP timeouts and a deadline. +## Async client + +`serpapi.AsyncClient` has the same SDK methods and API parameters as `serpapi.Client`, for programs that use `asyncio`, with some [transport differences](https://serpapi-python.readthedocs.io/en/latest/user_guide/async-client.html#differences-from-client). Call its request methods with `await`. It requires Python 3.11 or later and the `async` extra: + +```bash +pip install "serpapi[async]" +``` + +Run several searches at the same time with `asyncio.gather()`: + +```python +import asyncio +import os + +import serpapi + + +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + google, bing = await asyncio.gather( + client.search({"engine": "google", "q": "coffee"}), + client.search({"engine": "bing", "q": "coffee"}), + ) + + print("Google:", len(google.get("organic_results", [])), "results") + print("Bing:", len(bing.get("organic_results", [])), "results") + + +asyncio.run(main()) +``` + +`asyncio.gather()` returns the results in the order of the searches. `async with` closes the client's connections when the block ends. To fetch more pages, use `await results.next_page()` or `async for page in results.yield_pages()`. See [Migrating from Sync to Async](https://serpapi-python.readthedocs.io/en/latest/user_guide/async-client.html#migrating-from-sync-to-async) for a before-and-after example, or the [async client guide](https://serpapi-python.readthedocs.io/en/latest/user_guide/async-client.html) for limiting concurrent searches. + ## Examples Here are some examples of our most popular APIs. You can find the full list of supported engines and parameters in our [documentation](https://serpapi.com/search-engine-apis). diff --git a/docs/conf.py b/docs/conf.py index c7089ed..46a4f07 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ autodoc_default_options = { "members": True, "imported-members": True, - "exclude-members": "HTTPClient, from_http_response", + "exclude-members": "HTTPClient, AsyncHTTPClient, from_http_response", } html_theme = "sphinx_rtd_theme" diff --git a/docs/index.md b/docs/index.md index f416f44..7c2a8ab 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,7 +35,7 @@ uv add serpapi :::: -The package requires Python 3.6 or newer. +The package requires Python 3.6 or newer. To send requests with `asyncio` and `await`, install `serpapi[async]` on Python 3.11 or newer and use [`serpapi.AsyncClient`](user_guide/async-client.md). ## First Request @@ -95,6 +95,7 @@ Use the same search parameter names as the [SerpApi API documentation](https://s - [Parameters and Engines](user_guide/parameters-and-engines.md) explains search parameters and how to test them in the [SerpApi Playground](https://serpapi.com/playground). - For scripts that collect results, see [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). - For searches you retrieve later, selected response fields, and data retention settings, see [Async Search Archive](user_guide/async-search-archive.md), [JSON Restrictor](user_guide/json-restrictor.md), and [Zero Trace](user_guide/zero-trace.md). +- [Async Client](user_guide/async-client.md) covers `serpapi.AsyncClient`, which runs searches at the same time with `asyncio` and `await`. - [Examples](examples/index.md) includes searches for web pages, AI answers, local businesses, products, travel, finance, trends, jobs, media, apps, and research. ```{toctree} @@ -134,6 +135,7 @@ user_guide/migrating-from-google-search-results :caption: Advanced Usage user_guide/async-search-archive +user_guide/async-client user_guide/threading user_guide/multiprocessing user_guide/zero-trace diff --git a/docs/reference.md b/docs/reference.md index 9fbd9ce..47a776e 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -6,3 +6,7 @@ Look up the arguments, return values, and exceptions for the functions and class .. automodule:: serpapi :member-order: groupwise ``` + +```{eval-rst} +.. automethod:: serpapi.AsyncClient.aclose +``` diff --git a/docs/user_guide/async-client.md b/docs/user_guide/async-client.md new file mode 100644 index 0000000..da0bc6f --- /dev/null +++ b/docs/user_guide/async-client.md @@ -0,0 +1,347 @@ +--- +title: "Async Client" +description: "Migrate from Client to AsyncClient, compare their behavior, and run SerpApi searches concurrently with asyncio." +--- + +# Async Client + +`serpapi.AsyncClient` is a version of `serpapi.Client` for programs that use Python's `asyncio`, such as FastAPI or aiohttp applications. It has the same SDK methods and API parameters, with the [transport differences](#differences-from-client) described below. Call its request methods with `await`. While one request waits for a response, your program can run other requests and tasks. + +Python's `asyncio` is separate from SerpApi's `async` search parameter, which submits a search to retrieve later. You can use that parameter with either client. See [Async Search Archive](async-search-archive.md). + +## Installation + +`AsyncClient` requires Python 3.11 or newer and the `async` extra, which installs the [httpx2](https://pypi.org/project/httpx2/) HTTP library: + +::::{tab-set} + +:::{tab-item} pip + +```bash +pip install "serpapi[async]" +``` + +::: + +:::{tab-item} uv project + +```bash +uv add "serpapi[async]" +``` + +::: + +:::{tab-item} uv environment + +```bash +uv pip install "serpapi[async]" +``` + +::: + +:::: + +`serpapi.Client` doesn't need the extra. It uses the `requests` library and supports Python 3.6 or newer. If httpx2 isn't installed, creating an `AsyncClient` raises `ImportError` with the command to install the extra: + +```text +serpapi.AsyncClient requires Python 3.11 or later and the httpx2 package. Install the async extra with: pip install "serpapi[async]" +``` + +## First Search + +After [setting your API key](getting-started.md#installation), run this example: + +```python +import asyncio +import os + +import serpapi + + +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + results = await client.search( + engine="google", + q="coffee", + location="Austin, Texas", + ) + + print(results["organic_results"][0]["link"]) + + +asyncio.run(main()) +``` + +`asyncio.run()` starts an event loop and runs `main()` in it. `async with` closes the client's connections at the end of the block. If you can't use `async with`, for example when your application creates the client at startup, call `await client.aclose()` when you no longer need the client. + +For JSON responses, the client returns a `serpapi.AsyncSerpResults` object, which you can read like a dictionary, as with `SerpResults`. With `output="md"` or `output="html"`, it returns a string. + +The other examples on this page use the same imports. + +## Migrating from Sync to Async + +You can keep using `Client` for synchronous code; installing the `async` extra does not replace it or remove `requests`. To migrate a call path: + +1. Use Python 3.11 or newer and [install `serpapi[async]`](#installation). +2. Replace `Client` with `AsyncClient`, and manage its lifetime with `async with` or `await client.aclose()`. +3. Make the calling function `async def` and add `await` to requests and `next_page()`. Callers of that function must also await it. Replace pagination's `for` with `async for`. +4. Keep API parameters, supported request options, result-field access, and SDK exception handlers. Review [Differences from Client](#differences-from-client) if you customize the HTTP session or rely on transport-specific behavior. + +These examples perform the same search and read up to two pages: + +::::{tab-set} + +:::{tab-item} Before: synchronous + +```python +import os + +import serpapi + + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20) +try: + results = client.search(engine="google", q="coffee", timeout=(3, 20)) + for page in results.yield_pages(max_pages=2): + print(len(page.get("organic_results", []))) +finally: + client.session.close() +``` + +::: + +:::{tab-item} After: asynchronous + +```python +import asyncio +import os + +import serpapi + + +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + results = await client.search(engine="google", q="coffee", timeout=(3, 20)) + async for page in results.yield_pages(max_pages=2): + print(len(page.get("organic_results", []))) + + +asyncio.run(main()) +``` + +::: + +:::: + +Only operations that fetch data need asynchronous calls. Reading `results["organic_results"]`, calling `results.get(...)` or `results.as_dict()`, and formatting results remain synchronous. Keep pagination inside the client's lifetime; it fetches additional pages through that client. + +Use `asyncio.run()` at a script's entry point. Inside an async web handler or a notebook with an event loop already running, use `await main()` instead; do not nest `asyncio.run()`. Share an application client within one event loop, finish its requests before closing it, and do not reuse it across separate `asyncio.run()` calls. + +Sequential `await` calls still send one request at a time. Use [concurrent searches](#run-searches-at-the-same-time) when requests are independent. If your code polls with `time.sleep()`, replace it with `await asyncio.sleep()` so it doesn't block the event loop. Module helpers such as `serpapi.search()` remain blocking and cannot be made asynchronous by adding `await`; use the client methods below instead. + +You do not need to add the API parameter `{"async": True}` to migrate. That parameter controls SerpApi's server-side search jobs, not Python's event loop. + +## Methods + +Each `Client` method has an `AsyncClient` coroutine with the same name and parameters: + +| `Client` | `AsyncClient` | +| --- | --- | +| `client.search(...)` | `await client.search(...)` | +| `client.extract(...)` | `await client.extract(...)` | +| `client.search_archive(...)` | `await client.search_archive(...)` | +| `client.upload_image(...)` | `await client.upload_image(...)` | +| `client.locations(...)` | `await client.locations(...)` | +| `client.account(...)` | `await client.account(...)` | +| `results.next_page()` | `await results.next_page()` | +| `for page in results.yield_pages():` | `async for page in results.yield_pages():` | + +Module functions such as `serpapi.search()` use a synchronous client. Create an `AsyncClient` to send asynchronous requests. + +This example checks your account while it uploads an image, then searches Google Lens with the uploaded image. Place an image named `image.png` in the directory where you run the script, or replace `image.png` with the path to your file: + +```python +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=30) as client: + account, upload = await asyncio.gather( + client.account(), + client.upload_image("image.png"), + ) + results = await client.search( + engine="google_lens", + image_id=upload["image_id"], + type="visual_matches", + hl="en", + ) + + print("Searches left:", account["total_searches_left"]) + for match in results.get("visual_matches", [])[:3]: + print(match.get("title")) + + +asyncio.run(main()) +``` + +`asyncio.gather()` runs the account request and the upload at the same time, and returns their results in order. + +## Run Searches at the Same Time + +Pass several searches to `asyncio.gather()` to send them at once: + +```python +async def main(): + countries = ["us", "gb", "ca"] + + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + pages = await asyncio.gather(*( + client.search(engine="google", q="best coffee beans", gl=country, hl="en") + for country in countries + )) + + for country, results in zip(countries, pages): + organic_results = results.get("organic_results", []) + first = organic_results[0] if organic_results else {} + print(country, first.get("title")) + + +asyncio.run(main()) +``` + +One client can send many requests at once and reuses its connections between them. Share one client rather than creating a client for each search. + +### Limit Concurrent Searches + +`asyncio.gather()` starts every search right away. For a long list of searches, use an `asyncio.Semaphore` to limit how many run at once. This example runs up to two searches at a time and reports failed searches without stopping the others: + +```python +async def search_country(client, limit, country): + async with limit: + return await client.search( + engine="google", q="best coffee beans", gl=country, hl="en" + ) + + +async def main(): + countries = ["us", "gb", "ca", "au", "in"] + limit = asyncio.Semaphore(2) + + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + pages = await asyncio.gather( + *(search_country(client, limit, country) for country in countries), + return_exceptions=True, + ) + + for country, results in zip(countries, pages): + if isinstance(results, Exception): + print("Failed:", country, results) + else: + print("Finished:", country, len(results.get("organic_results", []))) + + +asyncio.run(main()) +``` + +`return_exceptions=True` makes `gather()` return an exception in place of a failed search's results, so one failure doesn't stop the other searches. Start with a small limit and increase it based on your account's search limits and response times. + +Each HTTP connection pool opens up to 100 connections at once by default. With the shared numeric, tuple, or `None` timeout forms, additional requests wait for a free connection without a pool-wait timeout. Distinct proxy or TLS settings use separate pools, so use a semaphore when you need one concurrency limit across all requests. + +## Pagination + +`next_page()` and `yield_pages()` send requests with the client that made the search, so use them before the client is closed: + +```python +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + results = await client.search(engine="google", q="coffee", location="Austin, Texas") + + next_results = await results.next_page() + if next_results: + print(len(next_results.get("organic_results", []))) + + async for page in results.yield_pages(max_pages=3): + current = page.get("serpapi_pagination", {}).get("current") + print(current, len(page.get("organic_results", []))) + + +asyncio.run(main()) +``` + +`next_page()` returns `None` when there is no next page. Each page's URL comes from the previous page, so `yield_pages()` fetches one page at a time. To fetch several Google pages at once, pass each page's `start` value to `search()` with `asyncio.gather()`. See [Google `start` Offsets](pagination.md#google-start-offsets). + +## Request Options + +`AsyncClient` accepts the same [request option names](request-options.md) as `Client`: `timeout`, `proxies`, `verify`, `stream`, and `cert`, with the [differences below](#differences-from-client), including buffered responses regardless of `stream`. It uses requests-compatible URL, search-parameter, and form-data encoding. For example, set a default timeout for the client and override it for one search: + +```python +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=20) as client: + results = await client.search(engine="google", q="coffee", timeout=10) + + print(results["search_metadata"]["status"]) + + +asyncio.run(main()) +``` + +Both constructors accept `api_key` and `timeout`. Pass `proxies`, `verify`, `stream`, and `cert` to individual method calls, not to the constructor. + +As with `Client`, a timeout in seconds applies to connecting and to each wait for data, and a `(connect, read)` tuple sets them separately. A per-request `timeout=None` disables the timeout, including a client default. Per-request zero, negative, and boolean timeout values are rejected. + +Both clients use certifi's CA bundle by default. They read `REQUESTS_CA_BUNDLE` (or `CURL_CA_BUNDLE` as a fallback) per request; an explicit `verify` path or `verify=False` overrides those variables. `SSL_CERT_FILE` and `SSL_CERT_DIR` do not override certifi. CA bundle files, CA directories, and client certificates supplied as a PEM file or `(cert, key)` pair work with `AsyncClient`. + +The async client loads CA bundle and client-certificate contents into an SSL context when an option-specific session is created. If you replace either file at the same path, close the existing `AsyncClient` and create a new one before making another request so the new contents are loaded. + +HTTP/HTTPS proxy dictionaries use the same selection rules as `requests`, including credentials in proxy URLs, scheme/host-specific entries, environment proxies, and `NO_PROXY`. Proxy selection is repeated after redirects. See [Differences from Client](#differences-from-client) for the remaining transport differences. + +## Errors + +`AsyncClient` uses the SDK's `HTTPError`, `HTTPConnectionError`, and `TimeoutError` types for HTTP status, connection, and timeout failures, so these handlers can stay the same: + +```python +async def main(): + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"], timeout=10) as client: + try: + results = await client.search(engine="google", q="coffee") + except serpapi.TimeoutError: + print("The request timed out.") + except serpapi.HTTPConnectionError: + print("Could not connect to SerpApi.") + except serpapi.HTTPError as exc: + print(exc.status_code, exc.error) + else: + print(results["search_metadata"]["status"]) + + +asyncio.run(main()) +``` + +See [Errors and Timeouts](errors-and-timeouts.md) for what each exception means. + +## Event Loops + +Use each client in only one event loop. Each `asyncio.run()` call starts a new event loop, so create the client inside the coroutine you pass to `asyncio.run()`, as the examples on this page do. In a long-running application, create one client when the application starts, share it, and close it with `await client.aclose()` when the application stops. + +`AsyncClient` runs on `asyncio` only. Other async libraries that httpx2 can run on, such as Trio, are not supported. + +## Differences from Client + +`AsyncClient` sends requests with httpx2 instead of `requests`; it is not a drop-in replacement for a customized `requests.Session`. Both clients retain the same API parameter names, dictionary-style result access, certifi-based default trust, HTTP/HTTPS proxy-selection rules, and common SDK exception types. + +| Area | `Client` | `AsyncClient` | +| --- | --- | --- | +| Installation | Base `serpapi` package; Python 3.6+. | `serpapi[async]`; Python 3.11+. | +| Execution and lifetime | Blocking requests; close the underlying session with `client.session.close()`. | Await requests inside one asyncio event loop; use `async with` or `await client.aclose()`. A closed client cannot be reused. | +| Results and pagination | `SerpResults`; `next_page()` and regular iteration. | `AsyncSerpResults`; `await next_page()` and `async for`. Dictionary helpers remain synchronous. | +| HTTP session | `requests.Session`. | `httpx2.AsyncClient`. Requests adapters, hooks, authentication objects, and session customization are not interchangeable. | +| Timeouts | Numbers, `None`, `(connect, read)` tuples, and requests/urllib3 timeout objects. | The same numbers, `None`, and tuples; also native `httpx2.Timeout`, but not requests/urllib3 timeout objects. | +| TLS contexts | Shared options include CA files/directories and client certificates. | Also accepts an `ssl.SSLContext` for `verify`. Load client certificates into that context; combining it with `cert` raises `TypeError`. The context is also used to connect to an HTTPS proxy, which is then offered a client certificate loaded into it. | +| SOCKS proxies | Requires requests' SOCKS dependencies; `socks5://` resolves DNS locally, `socks5h://` at the proxy. | Requires `pip install "httpx2[socks]"` separately. Both schemes resolve DNS at the proxy. Use `socks5h://` for matching behavior. SOCKS4 is unsupported. | +| Connection limits | The default pool cache size is not a hard concurrency limit. | Each pool defaults to at most 100 connections; this is not a global SDK limit. | +| Streaming | `stream` is forwarded to requests, but high-level methods still consume the response before returning results. | `stream` is accepted but ignored. Responses and file/iterator request bodies are buffered; file reading and body preparation are synchronous. | +| Error details | HTTP error responses are `requests.Response` objects. | HTTP error responses are `httpx2.Response` objects. Exception text can differ; blank read/write/pool timeout messages receive a fallback. | + +Neither client sets a timeout by default. A numeric or `(connect, read)` timeout is not a total wall-clock deadline, and the async client does not apply those forms to waiting for a free pool connection. Use a native `httpx2.Timeout` if you need a separate pool-wait timeout. + +The async client keeps its default pool plus up to 16 cached proxy/TLS/certificate combinations. Further uncached combinations use temporary sessions that close after the request, so they do not reuse connections between requests. Cached entries are not evicted. This bounds the number of cached pools but can reduce performance when rotating through many configurations. + +Async `HTTPError` messages omit the `api_key` query parameter, and httpx2 INFO request logs redact it. This does not sanitize raw request/response URLs; do not log those with credentials. Other transport failures, such as too many redirects, can still raise httpx2-specific exceptions rather than SDK exceptions. diff --git a/docs/user_guide/async-search-archive.md b/docs/user_guide/async-search-archive.md index 206cbd8..f81b79a 100644 --- a/docs/user_guide/async-search-archive.md +++ b/docs/user_guide/async-search-archive.md @@ -5,7 +5,7 @@ description: "Submit searches now and retrieve their results later with the Sear # Async Search Archive -An asynchronous search lets you submit a request now and collect the results later. Set the API's `async` parameter to `true` to receive a search ID, then use that ID with the Search Archive API. Each Python client call still waits for an HTTP response. These methods do not use Python's `asyncio` or `await`. +An asynchronous search lets you submit a request now and collect the results later. Set the API's `async` parameter to `true` to receive a search ID, then use that ID with the Search Archive API. The `async` parameter is separate from Python's `asyncio`. Each `serpapi.Client` call still waits for its HTTP response. To send requests with `asyncio` and `await`, use [`serpapi.AsyncClient`](async-client.md), which also accepts the `async` parameter. Create a client as shown in the [client usage guide](client-usage.md#create-a-client) before running the examples. diff --git a/docs/user_guide/client-usage.md b/docs/user_guide/client-usage.md index eccd43b..554a5de 100644 --- a/docs/user_guide/client-usage.md +++ b/docs/user_guide/client-usage.md @@ -5,7 +5,9 @@ description: "Use the SerpApi client, module helpers, request options, and respo # Client Usage -Create a `serpapi.Client` to reuse your API key, timeout, and HTTP connection settings across requests. +Create a `serpapi.Client` to reuse your API key, timeout, and HTTP connection settings across requests. If your program uses `asyncio`, [`serpapi.AsyncClient`](async-client.md) has the same methods for use with `await`. + +See [Migrating from Sync to Async](async-client.md#migrating-from-sync-to-async) for a before-and-after example and the [current client differences](async-client.md#differences-from-client). ## Create a Client @@ -86,7 +88,7 @@ locations = client.locations(q="Austin", limit=3) ## Request Options -`search()`, `search_archive()`, `account()`, and `locations()` pass these keyword arguments to the underlying `requests` call: +`search()`, `extract()`, `search_archive()`, `upload_image()`, `account()`, and `locations()` pass these keyword arguments to the underlying `requests` call: | Option | Use | | --- | --- | diff --git a/docs/user_guide/errors-and-timeouts.md b/docs/user_guide/errors-and-timeouts.md index 58a2814..9fbc76d 100644 --- a/docs/user_guide/errors-and-timeouts.md +++ b/docs/user_guide/errors-and-timeouts.md @@ -7,7 +7,7 @@ description: "Handle SerpApi HTTP errors, connection errors, timeouts, and API-l The client raises exceptions when a request times out, cannot connect, or receives an HTTP error. Use `try` and `except` to decide what your script should do when a request fails. -The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). +The examples use a client created as shown in [Client Usage](client-usage.md#create-a-client). `serpapi.AsyncClient` raises the same exceptions. See [Async Client](async-client.md#errors). ## HTTP Errors diff --git a/docs/user_guide/getting-started.md b/docs/user_guide/getting-started.md index 99490b1..5e0fea5 100644 --- a/docs/user_guide/getting-started.md +++ b/docs/user_guide/getting-started.md @@ -39,7 +39,7 @@ uv pip install serpapi :::: -The package requires Python 3.6 or newer. +The package requires Python 3.6 or newer. To send requests with `asyncio` and `await`, install the `async` extra on Python 3.11 or newer, as described in [Async Client](async-client.md#installation). Create or sign in to your SerpApi account and copy your API key from the [dashboard](https://serpapi.com/manage-api-key). An API key identifies your account when you make a request. Replace `secret_api_key` below with your key and run the command in your terminal. On Windows, use PowerShell. @@ -133,4 +133,5 @@ results = client.search(engine="google", q="coffee") - Read [Client Usage](client-usage.md) for response helpers, request options, and archive helpers. - Read [Pagination](pagination.md) when collecting more than one page of results. +- Read [Async Client](async-client.md) if your program uses `asyncio`. - Try [Google Across Countries](../examples/google-across-countries.md) to compare results from different countries. diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 4e58543..d9471b8 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -23,6 +23,7 @@ Start with [Getting Started](getting-started.md) to install the package and run ## Advanced Usage - [Async Search Archive](async-search-archive.md) +- [Async Client](async-client.md) - [Threading](threading.md) - [Multiprocessing](multiprocessing.md) - [Zero Trace and Data Retention](zero-trace.md) diff --git a/docs/user_guide/request-options.md b/docs/user_guide/request-options.md index 60c70db..0f86342 100644 --- a/docs/user_guide/request-options.md +++ b/docs/user_guide/request-options.md @@ -21,7 +21,9 @@ The client passes these keyword arguments to `requests.Session.request()`: | `stream` | Set the `requests` streaming option. The client still reads the response before returning results. | | `cert` | Authenticate the request with a client certificate. | -These options are supported by `search()`, `search_archive()`, `account()`, and `locations()`. +These options are supported by `search()`, `extract()`, `search_archive()`, `upload_image()`, `account()`, and `locations()`. + +`serpapi.AsyncClient` accepts the same options, including certifi-based TLS verification and HTTP/HTTPS proxy selection. It sends requests with httpx2 instead of `requests`; raw sessions, streaming, connection pools, and SOCKS DNS are not identical. See [Differences from Client](async-client.md#differences-from-client). ## Per-Request Timeout diff --git a/docs/user_guide/threading.md b/docs/user_guide/threading.md index 7b7259b..8cf4359 100644 --- a/docs/user_guide/threading.md +++ b/docs/user_guide/threading.md @@ -5,7 +5,7 @@ description: "Run several independent searches at the same time with ThreadPoolE # Threading -Search requests spend time waiting for network responses. Threads let your program make another request while one is waiting. Use them for independent searches, such as running the same query in several countries. +Search requests spend time waiting for network responses. Threads let your program make another request while one is waiting. Use them for independent searches, such as running the same query in several countries. If your program uses `asyncio`, run searches at the same time with [`serpapi.AsyncClient`](async-client.md) instead. ## ThreadPoolExecutor Example diff --git a/pyproject.toml b/pyproject.toml index 73c7799..86b85b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,8 @@ classifiers = [ [project.optional-dependencies] color = ["pygments"] -test = ["pytest"] +async = ["httpx2>=2.15.1; python_version >= '3.11'"] +test = ["pytest", "httpx2>=2.15.1; python_version >= '3.11'"] docs = [ "sphinx>=8.2,<9; python_version >= '3.11'", "myst-parser>=4,<5; python_version >= '3.11'", diff --git a/serpapi/core.py b/serpapi/core.py index f00f7cd..1fe9ba3 100644 --- a/serpapi/core.py +++ b/serpapi/core.py @@ -1,9 +1,13 @@ import io +import json import os -from .http import HTTPClient +import requests + +from .http import AsyncHTTPClient, HTTPClient from .exceptions import SearchIDNotProvided -from .models import SerpResults +from .models import AsyncSerpResults, SerpResults +from .utils import _response_json class Client(HTTPClient): @@ -213,6 +217,210 @@ def account(self, params: dict = None, **kwargs): return r.json() +def _pop_request_options(kwargs): + # These are arguments that should be passed to the underlying HTTP request. + request_kwargs = {} + for key in ["timeout", "proxies", "verify", "stream", "cert"]: + if key in kwargs: + request_kwargs[key] = kwargs.pop(key) + + return request_kwargs + + +def _async_response_json(response): + try: + return _response_json(response) + except json.JSONDecodeError as error: + raise requests.exceptions.JSONDecodeError( + error.msg, error.doc, error.pos + ) from error + + +class AsyncClient(AsyncHTTPClient): + """An asynchronous version of :class:`Client ` for use with ``await``. + + It has the same methods, parameters, and request options as ``Client``, and + each method is a coroutine. ``AsyncClient`` needs Python 3.11 or later and + the ``async`` extra: ``pip install "serpapi[async]"``. It sends requests with + `httpx2 `_ instead of ``requests``. + + .. code-block:: python + + async with serpapi.AsyncClient(api_key=os.environ["SERPAPI_KEY"]) as client: + results = await client.search(engine="google", q="coffee") + + Use the client in an ``async with`` block, or call ``await client.aclose()`` + when you are done with it, to close its connections. Use each client in a + single event loop. You can send several requests at once with the same client. + + :param api_key: The API Key to use for SerpApi.com. + :param timeout: The default timeout to use for requests. + """ + + DASHBOARD_URL = "https://serpapi.com/dashboard" + + def __init__(self, *, api_key=None, timeout=None): + super().__init__(api_key=api_key, timeout=timeout) + + def __repr__(self): + return "" + + async def search(self, params: dict = None, **kwargs): + """Fetch a page of results from SerpApi. Use it with ``await``. + + Returns a ``serpapi.AsyncSerpResults`` object for JSON responses, or text + when ``output="html"`` or ``output="md"`` is requested. Accepts the same + parameters as :meth:`Client.search `. + + :param params: Optional mapping of SerpApi search parameters such as ``engine``, ``q``, ``location``, and ``output``. + :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + + **Learn more**: https://serpapi.com/search-api + """ + if params is None: + params = {} + + request_kwargs = _pop_request_options(kwargs) + + if kwargs: + params.update(kwargs) + + r = await self.request("GET", "/search", params=params, **request_kwargs) + + return AsyncSerpResults.from_http_response(r, client=self) + + async def extract(self, params: dict = None, **kwargs): + """Extract content from a public URL using the Extract API. Use it with ``await``. + + Accepts the same parameters as :meth:`Client.extract `. + Sets ``engine=extract`` and returns a :class:`AsyncSerpResults ` object for JSON, or text for HTML and Markdown. + + :param params: Optional mapping of Extract API parameters, including the required ``url``. Put ``timeout`` here to set the API fetch timeout in milliseconds. + :param kwargs: Additional API parameters or HTTP request options, as in :meth:`search`. The ``timeout`` keyword controls the HTTP client timeout in seconds. + """ + params = dict(params) if params is not None else {} + kwargs["engine"] = "extract" + return await self.search(params, **kwargs) + + async def search_archive(self, params: dict = None, **kwargs): + """Get a result from the SerpApi Search Archive API. Use it with ``await``. + + :param params: Archive parameters. Must include ``search_id``. ``output`` accepts ``json`` (default), ``html``, or ``md``. + :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + + **Learn more**: https://serpapi.com/search-archive-api + """ + if params is None: + params = {} + + request_kwargs = _pop_request_options(kwargs) + + if kwargs: + params.update(kwargs) + + try: + search_id = params["search_id"] + except KeyError: + raise SearchIDNotProvided( + f"Please provide 'search_id', found here: { self.DASHBOARD_URL }" + ) + + r = await self.request( + "GET", f"/searches/{ search_id }", params=params, **request_kwargs + ) + return AsyncSerpResults.from_http_response(r, client=self) + + async def upload_image(self, image, **kwargs): + """Upload an image to SerpApi's Image API. Use it with ``await``. + + ``image`` can be a filesystem path or an open binary file object. The + returned dictionary contains an ``image_id`` that can be passed to + :meth:`search` for engines that accept uploaded images, such as Google + Lens. + + :param image: a path or open binary file object containing a JPG/JPEG, + PNG, or WebP image no larger than 500 KB. + :param api_key: the API Key to use for SerpApi.com. + :param **: any additional multipart form fields to pass to the API. + + **Learn more**: https://serpapi.com/image-api + """ + request_kwargs = _pop_request_options(kwargs) + + data = kwargs + if "api_key" not in data: + data["api_key"] = self.api_key + + image_file = None + try: + if isinstance(image, (str, os.PathLike)): + image_file = open(image, "rb") + image = image_file + elif isinstance(image, io.TextIOBase): + raise TypeError( + "image file must be opened in binary mode, e.g. open(path, 'rb')" + ) + + r = await self.request( + "POST", + "/image", + params={}, + data=data, + files={"image": image}, + **request_kwargs, + ) + return _async_response_json(r) + finally: + if image_file is not None: + image_file.close() + + async def locations(self, params: dict = None, **kwargs): + """Get a list of supported Google locations. Use it with ``await``. + + :param params: Location API parameters such as ``q`` and ``limit``. + :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + + **Learn more**: https://serpapi.com/locations-api + """ + if params is None: + params = {} + + request_kwargs = _pop_request_options(kwargs) + + if kwargs: + params.update(kwargs) + + r = await self.request( + "GET", + "/locations.json", + params=params, + assert_200=True, + **request_kwargs, + ) + return _async_response_json(r) + + async def account(self, params: dict = None, **kwargs): + """Get SerpApi account information. Use it with ``await``. + + :param params: Account API parameters. + :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + + **Learn more**: https://serpapi.com/account-api + """ + if params is None: + params = {} + + request_kwargs = _pop_request_options(kwargs) + + if kwargs: + params.update(kwargs) + + r = await self.request( + "GET", "/account.json", params=params, assert_200=True, **request_kwargs + ) + return _async_response_json(r) + + # An un-authenticated client instance. _client = Client() search = _client.search diff --git a/serpapi/exceptions.py b/serpapi/exceptions.py index c268c4d..23ec584 100644 --- a/serpapi/exceptions.py +++ b/serpapi/exceptions.py @@ -1,5 +1,7 @@ import requests +from .utils import _response_json + class SerpApiError(Exception): """Base class for exceptions in this module.""" @@ -27,9 +29,10 @@ def __init__(self, original_exception): http_error_exception: requests.exceptions.HTTPError = original_exception self.status_code = http_error_exception.response.status_code + # requests and httpx2 raise different JSON decoding errors; both are ValueErrors. try: - self.error = http_error_exception.response.json().get("error", None) - except requests.exceptions.JSONDecodeError: + self.error = _response_json(http_error_exception.response).get("error", None) + except ValueError: self.error = None else: self.status_code = -1 diff --git a/serpapi/http.py b/serpapi/http.py index 4ea1a33..66be1ef 100644 --- a/serpapi/http.py +++ b/serpapi/http.py @@ -1,3 +1,9 @@ +import asyncio +import logging +import os +import re +import ssl + import requests from .exceptions import ( @@ -69,3 +75,372 @@ def raise_for_status(r): r.raise_for_status() except requests.exceptions.HTTPError as e: raise HTTPError(e) + + +def _import_httpx2(): + try: + import httpx2 + except ImportError as e: + raise ImportError( + "serpapi.AsyncClient requires Python 3.11 or later and the httpx2 package. " + 'Install the async extra with: pip install "serpapi[async]"' + ) from e + + return httpx2 + + +class _ApiKeyLogFilter(logging.Filter): + _API_KEY_IN_URL = re.compile(r"(?i)([?&]api_key=)[^&\s\"']*") + + def filter(self, record): + message = record.getMessage() + redacted = self._API_KEY_IN_URL.sub(r"\1", message) + if redacted != message: + record.msg = redacted + record.args = () + return True + + +def _protect_httpx2_logs(): + logger = logging.getLogger("httpx2") + if not any(isinstance(item, _ApiKeyLogFilter) for item in logger.filters): + logger.addFilter(_ApiKeyLogFilter()) + + +class AsyncHTTPClient: + """This class handles outgoing asynchronous HTTP requests to SerpApi.com. + + It uses an ``httpx2.AsyncClient`` and accepts the same request options as + :class:`HTTPClient`, which uses ``requests``. + """ + + BASE_DOMAIN = HTTPClient.BASE_DOMAIN + USER_AGENT = HTTPClient.USER_AGENT + + # The most HTTP sessions kept open for requests with their own proxies, verify, or cert options. + _MAX_OPTION_SESSIONS = 16 + + def __init__(self, *, api_key=None, timeout=None): + self._httpx2 = _import_httpx2() + _protect_httpx2_logs() + self.api_key = api_key + self.timeout = timeout + self._closed = False + self._close_task = None + self._option_sessions = {} + self.session = self._create_session(verify=True) + + async def request(self, method, path, params, *, assert_200=True, **kwargs): + httpx2 = self._httpx2 + if self._closed: + raise RuntimeError("Cannot send a request, as the client has been closed.") + + # Inject the API Key into the params. + request_data = kwargs.get("data") + api_key_in_data = isinstance(request_data, dict) and "api_key" in request_data + if "api_key" not in params and not api_key_in_data: + params["api_key"] = self.api_key + + # Build the URL, as needed. + if not path.startswith("http"): + url = self.BASE_DOMAIN + path + else: + url = path + + headers = {"User-Agent": self.USER_AGENT} + + # Use the default timeout if one was provided to the client. + if self.timeout and "timeout" not in kwargs: + kwargs["timeout"] = self.timeout + if "timeout" in kwargs: + kwargs["timeout"] = _httpx2_timeout(httpx2, kwargs["timeout"]) + + # Translate the requests options that httpx2 names or handles differently. + if "allow_redirects" in kwargs: + kwargs["follow_redirects"] = kwargs.pop("allow_redirects") + follow_redirects = kwargs.pop("follow_redirects", True) + # Responses are always read in full, so stream has no effect. + kwargs.pop("stream", None) + proxies = kwargs.pop("proxies", None) + verify = kwargs.pop("verify", None) + cert = kwargs.pop("cert", None) + + # Encode the URL and body with requests, so they match what HTTPClient sends. + # httpx2 would replace the query string of a next page URL with the params, + # and encodes True, False, None, and file uploads differently. + prepared = requests.PreparedRequest() + prepared.prepare_method(method) + prepared.prepare_url(url, params) + prepared.prepare_headers(None) + prepared.prepare_body( + kwargs.pop("data", None) or {}, + kwargs.pop("files", None), + kwargs.pop("json", None), + ) + if "Content-Type" in prepared.headers: + headers["Content-Type"] = prepared.headers["Content-Type"] + if prepared.body is not None: + kwargs["content"] = _read_body(prepared.body) + + # Make the HTTP request. + temporary = False + try: + session, temporary = self._session_for(prepared.url, proxies, verify, cert) + r = await session.request( + method=method, url=prepared.url, headers=headers, + follow_redirects=False, **kwargs + ) + history = [] + while follow_redirects and r.next_request is not None: + if len(history) >= requests.models.DEFAULT_REDIRECT_LIMIT: + raise httpx2.TooManyRedirects( + "Exceeded maximum allowed redirects.", request=r.next_request + ) + history.append(r) + next_request = r.next_request + if temporary: + await session.aclose() + temporary = False + # Redirects may change the host or scheme and therefore the selected proxy. + session, temporary = self._session_for( + str(next_request.url), proxies, verify, cert + ) + r = await session.send(next_request, follow_redirects=False) + r.history = list(history) + except httpx2.ConnectTimeout as e: + # requests reports connection timeouts as connection errors. + raise _connection_error(e) from e + except httpx2.TimeoutException as e: + raise TimeoutError(str(e) or "The request timed out.") from e + except ( + httpx2.NetworkError, httpx2.ProtocolError, httpx2.ProxyError, ssl.SSLError + ) as e: + raise _connection_error(e) from e + finally: + if temporary: + await session.aclose() + + # Raise an exception if the status code is not 200. + if assert_200 and 400 <= r.status_code < 600: + raise _status_error(r) + + return r + + async def aclose(self): + """Close the client's HTTP connections. The client cannot send requests afterwards.""" + + self._closed = True + if self._close_task is None: + self._close_task = asyncio.create_task(self._close_sessions()) + + close_task = self._close_task + try: + await asyncio.shield(close_task) + except asyncio.CancelledError: + if close_task.done(): + self._close_task = None + raise + except Exception: + self._close_task = None + raise + else: + self._option_sessions.clear() + self._close_task = None + + async def _close_sessions(self): + sessions = [self.session, *self._option_sessions.values()] + results = await asyncio.gather( + *(session.aclose() for session in sessions), return_exceptions=True + ) + for result in results: + if isinstance(result, asyncio.CancelledError): + raise result + if isinstance(result, Exception): + raise result + + async def __aenter__(self): + if self._closed: + raise RuntimeError("Cannot reopen a client instance, once it has been closed.") + + return self + + async def __aexit__(self, exc_type=None, exc_value=None, traceback=None): + await self.aclose() + + def _session_for(self, url, proxies, verify, cert): + """Return the session for these request options, and whether to close it after the request.""" + + if self._closed: + raise RuntimeError("Cannot create a request session, as the client has been closed.") + + if verify is True or verify is None: + # Resolve environment settings per request, before choosing a cached session. + verify = ( + os.environ.get("REQUESTS_CA_BUNDLE") + or os.environ.get("CURL_CA_BUNDLE") + or True + ) + elif not isinstance(verify, ssl.SSLContext): + verify = os.fspath(verify) if verify else False + if not cert: + cert = None + proxy = _select_proxy(url, proxies or {}) + if proxy is None and verify is True and cert is None: + return self.session, False + + if cert is None or isinstance(cert, (str, bytes, os.PathLike)): + cert_key = cert + else: + cert_key = tuple(cert) + key = (proxy, verify, cert_key) + + session = self._option_sessions.get(key) + if session is not None: + return session, False + + session = self._create_session(proxy, verify, cert) + if len(self._option_sessions) >= self._MAX_OPTION_SESSIONS: + return session, True + + self._option_sessions[key] = session + return session, False + + def _create_session(self, proxy=None, verify=None, cert=None): + httpx2 = self._httpx2 + context = _ssl_context(httpx2, verify, cert) + if proxy and proxy.startswith("https://"): + # Like requests, verify the proxy with the request's settings, but only + # offer the client certificate to the destination. + proxy_context = _ssl_context(httpx2, verify, None) if cert else context + proxy = httpx2.Proxy(proxy, ssl_context=proxy_context) + + return httpx2.AsyncClient( + proxy=proxy, + # Proxy selection and CA environment overrides use requests' rules instead. + trust_env=False, + verify=context, + cookies=self.session.cookies.jar if hasattr(self, "session") else None, + timeout=None, + follow_redirects=True, + ) + + +def _read_body(body): + """Return a request body that httpx2 can send asynchronously. + + requests streams file objects and iterators, but httpx2's AsyncClient only + streams asynchronous iterators, so read them in full. + """ + + if isinstance(body, (str, bytes)): + return body + if hasattr(body, "read"): + return body.read() + if isinstance(body, (bytearray, memoryview)): + return bytes(body) + + return b"".join( + chunk.encode("utf-8") if isinstance(chunk, str) else chunk for chunk in body + ) + + +def _httpx2_timeout(httpx2, timeout): + """Convert a requests timeout to an httpx2 timeout. + + requests applies a number to connecting and reading, and a ``(connect, read)`` + tuple to each one. It uses the connect timeout while sending the request, + and doesn't limit the wait for a connection from the pool. + """ + + if isinstance(timeout, httpx2.Timeout): + return timeout + if isinstance(timeout, tuple): + try: + connect, read = timeout + except ValueError: + raise ValueError( + f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " + "or a single float to set both timeouts to the same value." + ) + else: + connect = read = timeout + + # Use requests' validation as httpx2 accepts zero, negative and boolean timeouts. + requests.adapters.TimeoutSauce(connect=connect, read=read) + return httpx2.Timeout(connect=connect, read=read, write=connect, pool=None) + + +def _select_proxy(url, proxies): + """Choose the proxy for a URL as requests does. Returns None to connect directly.""" + + proxies = dict(proxies) + no_proxy = proxies.get("no_proxy") + for scheme, proxy in requests.utils.get_environ_proxies(url, no_proxy=no_proxy).items(): + proxies.setdefault(scheme, proxy) + + # A proxy set to None disables the proxy from the environment for that scheme. + proxies = {key: proxy for key, proxy in proxies.items() if proxy is not None} + proxy = requests.utils.select_proxy(url, proxies) + if proxy: + return requests.utils.prepend_scheme_if_needed(proxy, "http") + + +def _ssl_context(httpx2, verify, cert): + """Build the SSL context for requests-style verify and cert options.""" + + if isinstance(verify, ssl.SSLContext): + if cert: + raise TypeError( + "cert can't be combined with an SSLContext for verify. " + "Load the certificate into the SSLContext with load_cert_chain() instead." + ) + return verify + + if verify is None: + # Like requests, use the CA bundle set in the environment, if any. + verify = ( + os.environ.get("REQUESTS_CA_BUNDLE") + or os.environ.get("CURL_CA_BUNDLE") + or True + ) + + if verify is True: + context = ssl.create_default_context(cafile=requests.certs.where()) + elif not verify: + context = httpx2.create_ssl_context(verify=False, trust_env=False) + elif not os.path.exists(verify): + raise OSError( + f"Could not find a suitable TLS CA certificate bundle, invalid path: {verify}" + ) + elif os.path.isdir(verify): + context = ssl.create_default_context(capath=verify) + else: + context = ssl.create_default_context(cafile=verify) + + if cert: + if isinstance(cert, (str, bytes, os.PathLike)): + context.load_cert_chain(cert) + else: + context.load_cert_chain(*cert) + + return context + + +def _connection_error(e): + # httpx2 exceptions raise RuntimeError, rather than returning None, when they have no request. + try: + request = getattr(e, "request", None) + except RuntimeError: + request = None + + return HTTPConnectionError(requests.exceptions.ConnectionError(*e.args, request=request)) + + +def _status_error(r): + """Build the HTTPError requests would raise for an error status, without the API key in its URL.""" + + kind = "Client" if r.status_code < 500 else "Server" + url = r.url.copy_remove_param("api_key") + message = f"{r.status_code} {kind} Error: {r.reason_phrase} for url: {url}" + + return HTTPError(requests.exceptions.HTTPError(message, response=r, request=r.request)) diff --git a/serpapi/models.py b/serpapi/models.py index 3e8a982..18449ca 100644 --- a/serpapi/models.py +++ b/serpapi/models.py @@ -3,6 +3,7 @@ from collections import UserDict from .textui import prettify_json +from .utils import _response_json class SerpResults(UserDict): @@ -93,9 +94,48 @@ def from_http_response(cls, r, *, client=None): return r.text try: - cls = cls(r.json(), client=client) + cls = cls(_response_json(r), client=client) return cls except ValueError: # If the response is not JSON, return the raw text. return r.text + + +class AsyncSerpResults(SerpResults): + """A dictionary-like object that represents the results of a :class:`AsyncClient ` request. + + It works like :class:`SerpResults `, but :meth:`next_page` + is a coroutine and :meth:`yield_pages` is an asynchronous generator. + """ + + async def next_page(self): + """Return the next page of results, if any. Use it with ``await``.""" + + if self.next_page_url: + # Include support for the API key, as it is not included in the next page URL. + params = {"api_key": self.client.api_key} + + r = await self.client.request("GET", path=self.next_page_url, params=params) + return AsyncSerpResults.from_http_response(r, client=self.client) + + async def yield_pages(self, max_pages=1_000): + """An asynchronous generator that ``yield`` s the next ``n`` pages of search results, if any. + + Use it with ``async for``. + + :param max_pages: limit the number of pages yielded to ``n``. + """ + + current_page_count = 0 + + current_page = self + while current_page and current_page_count < max_pages: + yield current_page + current_page_count += 1 + if current_page_count >= max_pages: + break + if current_page.next_page_url: + current_page = await current_page.next_page() + else: + break diff --git a/serpapi/utils.py b/serpapi/utils.py index 4a94510..91182f5 100644 --- a/serpapi/utils.py +++ b/serpapi/utils.py @@ -1,5 +1,19 @@ +import json import os def api_key_from_environment(): return os.getenv("SERP_API_KEY") + + +def _response_json(response): + """Parse the JSON body of a requests or httpx2 response. + + httpx2 only decodes UTF-8, UTF-16 and UTF-32 bodies. Like requests, decode + any other body with the response's charset. + """ + + try: + return response.json() + except UnicodeDecodeError: + return json.loads(response.text) diff --git a/tests/docs_example_support.py b/tests/docs_example_support.py index 1d48679..d1daedc 100644 --- a/tests/docs_example_support.py +++ b/tests/docs_example_support.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager import json import os from pathlib import Path @@ -10,7 +11,7 @@ from urllib.parse import parse_qs, urlsplit import serpapi -from serpapi.http import HTTPClient +from serpapi.http import AsyncHTTPClient, HTTPClient ROOT = Path(__file__).resolve().parents[1] @@ -78,29 +79,44 @@ def validate_response(response, path, params): def install_http_checks(): original_request = HTTPClient.request + original_async_request = AsyncHTTPClient.request report_path = Path(os.environ["DOCS_EXAMPLE_REPORT_DIR"]) / f"{os.getpid()}.jsonl" lock = threading.Lock() - def checked_request(self, method, path, params, **kwargs): + @contextmanager + def checked(client, path, params, kwargs): + """Record one request, and yield a function that validates its response.""" url = urlsplit(path) query = {key: values[-1] for key, values in parse_qs(url.query).items()} query.update(params) record = {"path": url.path, "engine": query.get("engine"), "ok": False} - if not self.timeout and not kwargs.get("timeout"): + if not client.timeout and not kwargs.get("timeout"): kwargs["timeout"] = 30 try: - response = original_request(self, method, path, params, **kwargs) - validate_response(response, url.path, query) + yield lambda response: validate_response(response, url.path, query) record["ok"] = True - return response - except Exception as exc: + # BaseException also records requests that asyncio cancels. + except BaseException as exc: record["error"] = redact(f"{type(exc).__name__}: {exc}") raise finally: with lock, report_path.open("a") as report: report.write(json.dumps(record) + "\n") + def checked_request(self, method, path, params, **kwargs): + with checked(self, path, params, kwargs) as validate: + response = original_request(self, method, path, params, **kwargs) + validate(response) + return response + + async def checked_async_request(self, method, path, params, **kwargs): + with checked(self, path, params, kwargs) as validate: + response = await original_async_request(self, method, path, params, **kwargs) + validate(response) + return response + HTTPClient.request = checked_request + AsyncHTTPClient.request = checked_async_request def script_for_page(path): diff --git a/tests/test_async_client.py b/tests/test_async_client.py new file mode 100644 index 0000000..892a25b --- /dev/null +++ b/tests/test_async_client.py @@ -0,0 +1,635 @@ +import asyncio +import json +import logging +import ssl +from pathlib import Path + +import certifi +import pytest +import requests + +# AsyncClient needs the async extra, which is only available on Python 3.11 and later. +httpx2 = pytest.importorskip("httpx2") + +import serpapi # noqa: E402 +from serpapi.http import _select_proxy, _ssl_context # noqa: E402 + +pytestmark = pytest.mark.anyio + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +@pytest.fixture(autouse=True) +def clean_transport_environment(monkeypatch): + for name in ("http_proxy", "https_proxy", "all_proxy", "no_proxy"): + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv(name.upper(), raising=False) + for name in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR"): + monkeypatch.delenv(name, raising=False) + + +class Server: + """Answers and records the requests of every session an AsyncClient creates.""" + + def __init__(self): + self.handler = lambda request: httpx2.Response(200, json={}) + self.requests = [] + # The keyword arguments and httpx2.AsyncClient of each session. + self.sessions = [] + + async def handle(self, request): + self.requests.append(request) + response = self.handler(request) + if asyncio.iscoroutine(response): + response = await response + return response + + @property + def last_request(self): + return self.requests[-1] + + +@pytest.fixture +def server(monkeypatch): + server = Server() + real_async_client = httpx2.AsyncClient + + def async_client(**kwargs): + # Leave out the proxy and TLS options, so that requests go to the server instead of the network. + session = real_async_client( + transport=httpx2.MockTransport(server.handle), + follow_redirects=kwargs["follow_redirects"], + timeout=kwargs["timeout"], + cookies=kwargs.get("cookies"), + ) + server.sessions.append((kwargs, session)) + return session + + monkeypatch.setattr(httpx2, "AsyncClient", async_client) + return server + + +@pytest.fixture +async def async_client(server): + async with serpapi.AsyncClient(api_key="test-api-key") as client: + yield client + + +async def test_search_sends_the_same_request_as_client(async_client, server): + results = await async_client.search( + engine="google", q="coffee shops", no_cache=True, start=None, tbs=["a", "b"] + ) + + assert isinstance(results, serpapi.AsyncSerpResults) + assert results.client is async_client + assert str(server.last_request.url) == ( + "https://serpapi.com/search" + "?engine=google&q=coffee+shops&no_cache=True&tbs=a&tbs=b&api_key=test-api-key" + ) + assert server.last_request.headers["User-Agent"] == serpapi.HTTPClient.USER_AGENT + + +@pytest.mark.parametrize( + ("output", "content_type"), [("html", "text/html"), ("md", "text/markdown")] +) +async def test_search_returns_text_for_html_and_markdown( + async_client, server, output, content_type +): + server.handler = lambda request: httpx2.Response( + 200, text="

Coffee

", headers={"Content-Type": content_type} + ) + + assert await async_client.search(q="coffee", output=output) == "

Coffee

" + + +@pytest.mark.parametrize( + ("client_timeout", "request_options", "expected"), + [ + (None, {}, {"connect": None, "read": None, "write": None, "pool": None}), + (10, {}, {"connect": 10, "read": 10, "write": 10, "pool": None}), + (10, {"timeout": 5}, {"connect": 5, "read": 5, "write": 5, "pool": None}), + (10, {"timeout": (3, 7)}, {"connect": 3, "read": 7, "write": 3, "pool": None}), + (10, {"timeout": None}, {"connect": None, "read": None, "write": None, "pool": None}), + ], +) +async def test_timeouts_work_like_requests_timeouts( + server, client_timeout, request_options, expected +): + async with serpapi.AsyncClient(api_key="test-api-key", timeout=client_timeout) as client: + await client.search(q="coffee", **request_options) + + assert server.last_request.extensions["timeout"] == expected + + +async def test_extract_separates_api_and_http_timeouts(async_client, server): + params = {"url": "https://example.com", "timeout": 5000} + + await async_client.extract(params, timeout=3) + + assert server.last_request.url.params["engine"] == "extract" + assert server.last_request.url.params["timeout"] == "5000" + assert server.last_request.extensions["timeout"]["read"] == 3 + assert params == {"url": "https://example.com", "timeout": 5000} + + +async def test_search_archive(async_client, server): + with pytest.raises(serpapi.SearchIDNotProvided): + await async_client.search_archive() + + results = await async_client.search_archive(search_id="abc123") + + assert isinstance(results, serpapi.AsyncSerpResults) + assert server.last_request.url.path == "/searches/abc123" + + +async def test_upload_image_sends_multipart_request(async_client, server, tmp_path): + image_path = tmp_path / "test.png" + image_path.write_bytes(b"fake-png-data") + server.handler = lambda request: httpx2.Response(200, json={"image_id": "image-123"}) + + result = await async_client.upload_image(image_path, zero_trace=True, timeout=5) + + assert result == {"image_id": "image-123"} + request = server.last_request + assert request.method == "POST" + assert str(request.url) == "https://serpapi.com/image" + assert request.extensions["timeout"]["read"] == 5 + assert request.headers["Content-Type"].startswith("multipart/form-data; boundary=") + assert b'name="zero_trace"\r\n\r\nTrue\r\n' in request.content + assert b'name="api_key"\r\n\r\ntest-api-key\r\n' in request.content + assert b'name="image"; filename="test.png"\r\n\r\nfake-png-data\r\n' in request.content + + +async def test_upload_image_rejects_text_mode_file(async_client, server, tmp_path): + image_path = tmp_path / "test.png" + image_path.write_text("fake-png-data") + + with image_path.open("r") as image, pytest.raises(TypeError, match="binary mode"): + await async_client.upload_image(image) + + assert server.requests == [] + + +async def test_locations_and_account_return_json(async_client, server): + def handler(request): + if request.url.path == "/locations.json": + return httpx2.Response(200, json=[{"name": "Austin"}]) + return httpx2.Response(200, json={"plan_name": "Free"}) + + server.handler = handler + + assert await async_client.locations(q="Austin", limit=3) == [{"name": "Austin"}] + assert server.last_request.url.params["limit"] == "3" + assert await async_client.account() == {"plan_name": "Free"} + assert server.last_request.url.path == "/account.json" + + +async def test_error_status_raises_http_error_without_the_api_key(async_client, server): + server.handler = lambda request: httpx2.Response(401, json={"error": "Invalid API key."}) + + with pytest.raises(serpapi.HTTPError) as exc_info: + await async_client.search(q="coffee") + + assert exc_info.value.status_code == 401 + assert exc_info.value.error == "Invalid API key." + assert isinstance(exc_info.value.response, httpx2.Response) + assert str(exc_info.value) == ( + "401 Client Error: Unauthorized for url: https://serpapi.com/search?q=coffee" + ) + + response = await async_client.request("GET", "/search", {}, assert_200=False) + assert response.status_code == 401 + + +async def test_error_status_without_json_has_no_error_message(async_client, server): + server.handler = lambda request: httpx2.Response(503, text="Unavailable") + + with pytest.raises(serpapi.HTTPError) as exc_info: + await async_client.account() + + assert exc_info.value.status_code == 503 + assert exc_info.value.error is None + + +async def test_http_logs_redact_api_key_from_search_and_pagination( + async_client, server, caplog +): + server.handler = lambda request: httpx2.Response( + 200, + json={ + "serpapi_pagination": { + "next": "https://serpapi.com/search?start=10", + } + }, + ) + caplog.set_level(logging.INFO, logger="httpx2") + + results = await async_client.search(q="coffee") + await results.next_page() + + http_logs = [ + record.getMessage() + for record in caplog.records + if record.name == "httpx2" and record.getMessage().startswith("HTTP Request:") + ] + assert len(http_logs) == 2 + assert all("test-api-key" not in message for message in http_logs) + assert all("api_key=" in message for message in http_logs) + assert "q=coffee" in http_logs[0] + assert "start=10" in http_logs[1] + + +@pytest.mark.parametrize( + ("exception", "expected", "message", "expected_message"), + [ + (httpx2.ConnectTimeout, serpapi.HTTPConnectionError, "failed", "failed"), + (httpx2.ConnectError, serpapi.HTTPConnectionError, "failed", "failed"), + (httpx2.RemoteProtocolError, serpapi.HTTPConnectionError, "failed", "failed"), + (httpx2.ReadTimeout, serpapi.TimeoutError, "failed", "failed"), + (httpx2.ReadTimeout, serpapi.TimeoutError, "", "The request timed out."), + ], +) +async def test_transport_errors_raise_serpapi_errors( + async_client, server, exception, expected, message, expected_message +): + def handler(request): + raise exception(message, request=request) + + server.handler = handler + + with pytest.raises(expected) as exc_info: + await async_client.search(q="coffee") + + assert isinstance(exc_info.value.__cause__, exception) + if expected is serpapi.TimeoutError: + assert str(exc_info.value) == expected_message + + +@pytest.mark.parametrize("method", ["account", "locations", "upload_image"]) +@pytest.mark.parametrize( + "body", + [b"Proxy sign-in required", b"Acc\xe8s refus\xe9"], + ids=["utf-8", "latin-1"], +) +async def test_dictionary_methods_wrap_json_decode_errors( + async_client, server, method, body, tmp_path +): + server.handler = lambda request: httpx2.Response( + 200, content=body, headers={"Content-Type": "text/html"} + ) + arguments = () + if method == "upload_image": + image = tmp_path / "image.png" + image.write_bytes(b"image-bytes") + arguments = (image,) + + with pytest.raises(requests.exceptions.JSONDecodeError) as exc_info: + await getattr(async_client, method)(*arguments) + + assert isinstance(exc_info.value, requests.exceptions.RequestException) + assert isinstance(exc_info.value.__cause__, json.JSONDecodeError) + + +async def test_dictionary_methods_decode_json_with_the_response_charset(async_client, server): + server.handler = lambda request: httpx2.Response( + 200, + content=b'{"plan_name": "Caf\xe9"}', + headers={"Content-Type": "application/json; charset=iso-8859-1"}, + ) + + assert await async_client.account() == {"plan_name": "Caf\u00e9"} + + +async def test_results_decode_json_with_the_response_charset(async_client, server): + server.handler = lambda request: httpx2.Response( + 200, + content=( + b'{"title": "Caf\xe9", ' + b'"serpapi_pagination": {"next": "https://serpapi.com/search?start=10"}}' + ), + headers={"Content-Type": "application/json; charset=iso-8859-1"}, + ) + + results = await async_client.search(q="coffee") + next_results = await results.next_page() + + assert isinstance(results, serpapi.AsyncSerpResults) + assert isinstance(next_results, serpapi.AsyncSerpResults) + assert next_results["title"] == "Caf\u00e9" + + +async def test_error_status_decodes_json_with_the_response_charset(async_client, server): + server.handler = lambda request: httpx2.Response( + 401, + content=b'{"error": "Cl\xe9 API invalide"}', + headers={"Content-Type": "application/json; charset=iso-8859-1"}, + ) + + with pytest.raises(serpapi.HTTPError) as exc_info: + await async_client.search(q="coffee") + + assert exc_info.value.error == "Cl\u00e9 API invalide" + + +async def test_tls_alerts_raise_connection_errors(async_client, server): + def handler(request): + raise ssl.SSLError("TLS peer requires a client certificate") + + server.handler = handler + with pytest.raises(serpapi.HTTPConnectionError) as exc_info: + await async_client.search(q="coffee") + assert isinstance(exc_info.value.__cause__, ssl.SSLError) + + +@pytest.mark.parametrize("timeout", [0, -1, True, (1, 2, 3), (1, -1)]) +async def test_invalid_timeouts_are_rejected_like_requests(async_client, server, timeout): + with pytest.raises(ValueError): + await async_client.search(q="coffee", timeout=timeout) + assert not server.requests + + +@pytest.mark.parametrize( + ("max_pages", "available_pages", "expected_pages"), + [(1, 3, 1), (2, 3, 2), (3, 3, 3), (5, 3, 3), (5, 1, 1), (0, 3, 0)], +) +async def test_yield_pages_does_not_request_unused_pages( + async_client, server, max_pages, available_pages, expected_pages +): + def handler(request): + page_number = int(request.url.params.get("start", 0)) // 10 + 1 + data = {"search_information": {"page_number": page_number}} + if page_number < available_pages: + data["serpapi_pagination"] = { + "next": f"https://serpapi.com/search?engine=google&q=Coffee&start={page_number * 10}" + } + return httpx2.Response(200, json=data) + + server.handler = handler + results = await async_client.search(engine="google", q="Coffee") + + pages = [page async for page in results.yield_pages(max_pages=max_pages)] + + assert [page["search_information"]["page_number"] for page in pages] == list( + range(1, expected_pages + 1) + ) + assert len(server.requests) == max(1, expected_pages) + assert all(request.url.params["api_key"] == "test-api-key" for request in server.requests) + + +async def test_requests_run_concurrently(async_client, server): + arrived = [] + all_arrived = asyncio.Event() + + async def handler(request): + arrived.append(request) + if len(arrived) == 3: + all_arrived.set() + # Each response waits for all three requests, so they must be sent at the same time. + await all_arrived.wait() + return httpx2.Response(200, json={"q": request.url.params["q"]}) + + server.handler = handler + + results = await asyncio.wait_for( + asyncio.gather(*(async_client.search(q=q) for q in ["a", "b", "c"])), timeout=5 + ) + + assert [result["q"] for result in results] == ["a", "b", "c"] + + +async def test_request_options_use_cached_sessions(async_client, server): + proxies = {"https": "http://proxy.example:8080"} + + await async_client.search(q="coffee", proxies=proxies) + await async_client.search(q="tea", proxies=dict(proxies)) + await async_client.search(q="coffee", verify=False) + await async_client.search(q="coffee") + + assert len(server.sessions) == 3 + (default, _), (proxy, _), (no_verify, _) = server.sessions + assert default["proxy"] is None + assert default["trust_env"] is False + assert proxy["proxy"] == "http://proxy.example:8080" + assert proxy["trust_env"] is False + assert no_verify["verify"].verify_mode == ssl.CERT_NONE + for options, _ in server.sessions: + assert options["follow_redirects"] is True + assert options["timeout"] is None + + +async def test_sessions_beyond_the_cache_limit_close_after_the_request( + async_client, server, monkeypatch +): + monkeypatch.setattr(async_client, "_MAX_OPTION_SESSIONS", 1) + + await async_client.search(q="coffee", verify=False) + await async_client.search(q="coffee", proxies={"https": "http://proxy.example:8080"}) + + (_, default), (_, cached), (_, temporary) = server.sessions + assert not cached.is_closed + assert temporary.is_closed + + +async def test_closed_client_cannot_send_requests_or_reopen(server): + client = serpapi.AsyncClient(api_key="test-api-key") + await client.search(q="coffee", verify=False) + + await client.aclose() + await client.aclose() + + assert all(session.is_closed for _, session in server.sessions) + with pytest.raises(RuntimeError, match="closed"): + await client.search(q="coffee") + with pytest.raises(RuntimeError, match="reopen"): + async with client: + pass + + +async def test_aclose_finishes_closing_sessions_when_caller_is_cancelled( + server, monkeypatch +): + client = serpapi.AsyncClient(api_key="test-api-key") + await client.search(q="coffee", verify=False) + sessions = [client.session, *client._option_sessions.values()] + + close_started = asyncio.Event() + allow_close = asyncio.Event() + original_aclose = client.session.aclose + + async def delayed_close(): + close_started.set() + await allow_close.wait() + await original_aclose() + + monkeypatch.setattr(client.session, "aclose", delayed_close) + close_caller = asyncio.create_task(client.aclose()) + try: + await close_started.wait() + close_caller.cancel() + with pytest.raises(asyncio.CancelledError): + await close_caller + + allow_close.set() + await client.aclose() + assert all(session.is_closed for session in sessions) + finally: + allow_close.set() + await client.aclose() + + +async def test_closed_client_does_not_create_new_sessions(async_client): + await async_client.aclose() + + with pytest.raises(RuntimeError, match="closed"): + async_client._session_for( + "https://serpapi.com/search", + {"https": "http://proxy.example:8080"}, + None, + None, + ) + + +def test_select_proxy_merges_environment_proxies_like_requests(monkeypatch): + for name in ["http_proxy", "https_proxy", "all_proxy", "no_proxy"]: + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv(name.upper(), raising=False) + monkeypatch.setenv("HTTPS_PROXY", "env-proxy.example:3128") + url = "https://serpapi.com/search" + + assert _select_proxy(url, {"http": "http://other.example:8080"}) == ( + "http://env-proxy.example:3128" + ) + assert _select_proxy(url, {"https": "http://proxy.example:8080"}) == ( + "http://proxy.example:8080" + ) + assert _select_proxy(url, {"https": None}) is None + assert _select_proxy(url, {"http": "http://other.example:8080", "no_proxy": "serpapi.com"}) is None + + +def test_ssl_context_supports_requests_verify_options(monkeypatch): + assert _ssl_context(httpx2, False, None).verify_mode == ssl.CERT_NONE + assert _ssl_context(httpx2, certifi.where(), None).get_ca_certs() + + monkeypatch.setenv("REQUESTS_CA_BUNDLE", certifi.where()) + assert _ssl_context(httpx2, None, None).get_ca_certs() + + with pytest.raises(TypeError, match="load_cert_chain"): + _ssl_context(httpx2, ssl.create_default_context(), "client.pem") + + +def test_default_tls_uses_certifi_not_ssl_environment(monkeypatch, tmp_path): + monkeypatch.setenv("SSL_CERT_FILE", str(tmp_path / "missing.pem")) + monkeypatch.setenv("SSL_CERT_DIR", str(tmp_path / "missing-directory")) + + context = _ssl_context(httpx2, True, None) + expected = ssl.create_default_context(cafile=requests.certs.where()) + + assert context.verify_mode == ssl.CERT_REQUIRED + assert context.check_hostname + assert context.get_ca_certs(binary_form=True) == expected.get_ca_certs(binary_form=True) + + +async def test_proxy_environment_is_resolved_for_each_request(async_client, server, monkeypatch): + monkeypatch.setenv("HTTPS_PROXY", "http://first.example:8080") + await async_client.search(q="coffee") + assert server.sessions[-1][0]["proxy"] == "http://first.example:8080" + + monkeypatch.setenv("HTTPS_PROXY", "http://second.example:8080") + await async_client.search(q="tea") + assert server.sessions[-1][0]["proxy"] == "http://second.example:8080" + + monkeypatch.setenv("NO_PROXY", "serpapi.com") + await async_client.search(q="water") + assert len(server.sessions) == 3 + assert server.sessions[0][0]["trust_env"] is False + + +@pytest.mark.parametrize("variable", ["REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"]) +async def test_ca_environment_is_resolved_for_each_request( + async_client, server, monkeypatch, tmp_path, variable +): + bundle = tmp_path / "ca.pem" + bundle.write_bytes(Path(certifi.where()).read_bytes()) + monkeypatch.setenv(variable, str(bundle)) + + await async_client.search(q="coffee", verify=True) + + assert len(server.sessions) == 2 + assert server.sessions[-1][0]["verify"].get_ca_certs() + + monkeypatch.setenv(variable, str(tmp_path / "missing.pem")) + with pytest.raises(OSError): + await async_client.search(q="tea") + + +async def test_https_proxy_uses_the_request_tls_context(async_client, server): + await async_client.search(proxies={"https": "https://proxy.example:8443"}, verify=False) + + options, _ = server.sessions[-1] + assert isinstance(options["proxy"], httpx2.Proxy) + assert options["proxy"].ssl_context is options["verify"] + assert options["proxy"].ssl_context.verify_mode == ssl.CERT_NONE + + +async def test_https_proxy_is_not_offered_the_client_certificate( + async_client, server, monkeypatch +): + with_client_certificate = [] + monkeypatch.setattr( + ssl.SSLContext, + "load_cert_chain", + lambda context, *args, **kwargs: with_client_certificate.append(context), + ) + + await async_client.search( + proxies={"https": "https://proxy.example:8443"}, verify=False, cert="client.pem" + ) + + options, _ = server.sessions[-1] + assert with_client_certificate == [options["verify"]] + assert options["proxy"].ssl_context is not options["verify"] + assert options["proxy"].ssl_context.verify_mode == ssl.CERT_NONE + + +async def test_redirects_reselect_the_proxy_for_the_destination(async_client, server): + def handler(request): + if request.url.scheme == "http": + return httpx2.Response(302, headers={"Location": "https://serpapi.com/search"}) + return httpx2.Response(200, json={}) + + server.handler = handler + await async_client.request("GET", "http://serpapi.com/search", {}, proxies={ + "http": "http://first.example:8080", "https": "http://second.example:8080", + }) + assert [options["proxy"] for options, _ in server.sessions] == [ + None, "http://first.example:8080", "http://second.example:8080", + ] + + +@pytest.mark.parametrize("redirects", [25, 31]) +async def test_redirect_limit_matches_requests(async_client, server, redirects): + def handler(request): + remaining = int(request.url.params["redirects"]) + if remaining: + return httpx2.Response(302, headers={"Location": "/search?redirects={}".format(remaining - 1)}) + return httpx2.Response(200, json={"done": True}) + + server.handler = handler + if redirects > 30: + with pytest.raises(httpx2.TooManyRedirects): + await async_client.search(redirects=redirects) + else: + result = await async_client.search(redirects=redirects) + assert result["done"] + + +async def test_cookies_survive_request_option_changes(async_client, server): + server.handler = lambda request: httpx2.Response( + 200, json={}, headers={"Set-Cookie": "session=local-value; Path=/"} + ) + await async_client.search(q="coffee") + await async_client.search(q="tea", verify=False) + assert server.last_request.headers["Cookie"] == "session=local-value" diff --git a/tests/test_docs_example_runner.py b/tests/test_docs_example_runner.py index c0d5f8c..596118b 100644 --- a/tests/test_docs_example_runner.py +++ b/tests/test_docs_example_runner.py @@ -90,6 +90,44 @@ def test_doc_runner_fails_on_caught_api_errors_and_redacts_key(tmp_path): assert "private-test-key" not in report.read_text() +def test_doc_runner_checks_async_client_requests(tmp_path): + pytest.importorskip("httpx2") + code = """ +import asyncio +import httpx2 + +async def fake_send(self, request, **kwargs): + if request.url.params['engine'] == 'google': + payload = {'organic_results': [{'title': 'Coffee'}]} + else: + payload = {'error': 'No results'} + return httpx2.Response(200, json=payload, request=request) + +httpx2.AsyncClient.send = fake_send + +async def main(): + async with serpapi.AsyncClient(api_key='secret_api_key') as client: + results = await asyncio.gather( + client.search(engine='google', q='coffee'), + client.search(engine='bing', q='coffee'), + return_exceptions=True, + ) + assert results[0]['organic_results'][0]['title'] == 'Coffee' + assert isinstance(results[1], AssertionError) + +asyncio.run(main()) +""" + page = markdown_page(tmp_path, code) + with pytest.raises(RuntimeError, match="bing: AssertionError: No results"): + run_page(page, tmp_path / "run", "test-key") + records = [ + json.loads(line) + for report in (tmp_path / "run" / "requests").glob("*.jsonl") + for line in report.read_text().splitlines() + ] + assert {record["engine"]: record["ok"] for record in records} == {"google": True, "bing": False} + + def test_doc_runner_rejects_examples_without_requests(tmp_path): page = markdown_page(tmp_path, "if __name__ == 'docs_examples':\n client.search(q='coffee')") with pytest.raises(RuntimeError, match="without making a SerpApi request"): diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 02c0793..b33b45e 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,3 +1,5 @@ +import subprocess +import sys from unittest.mock import Mock import pytest import requests @@ -31,3 +33,16 @@ def test_transport_errors_are_translated(monkeypatch, cause, expected): with pytest.raises(expected): client.search(engine="google", q="coffee") + + +def test_async_client_explains_how_to_install_the_extra(monkeypatch): + monkeypatch.setitem(sys.modules, "httpx2", None) + + with pytest.raises(ImportError, match=r'pip install "serpapi\[async\]"'): + serpapi.AsyncClient(api_key="test-api-key") + + +def test_importing_serpapi_does_not_import_httpx2(): + code = "import sys, serpapi; assert 'httpx2' not in sys.modules" + + subprocess.run([sys.executable, "-c", code], check=True) From 4fe0f3fd4139d377642a81a726c74301531caa2d Mon Sep 17 00:00:00 2001 From: Adarsh Divakaran Date: Mon, 16 Nov 2026 00:56:24 +0530 Subject: [PATCH 2/4] CI: set up Python with uv in the test matrices ubuntu-latest is now Ubuntu 26.04, where actions/setup-python has no Python 3.8 or 3.9. uv downloads its own builds of every version in the matrix. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 8 ++++---- .github/workflows/release.yml | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d19c8e..5f3f584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,14 +21,14 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + - name: Set up uv with Python ${{ matrix.python-version }} + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} - + activate-environment: true - name: Install dependencies - run: pip install -e .[test] + run: uv pip install -e '.[test]' - name: Run tests run: pytest tests --ignore-glob='tests/test_docs_*.py' -k "not example" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e37094..3378816 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,12 +30,13 @@ jobs: echo "Release must run against a tag, got '${{ github.ref }}'" >&2 exit 1 - - uses: actions/setup-python@v6 + - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} + activate-environment: true - name: Install dependencies - run: pip install -e .[test] + run: uv pip install -e '.[test]' - name: Run tests run: pytest --ignore=tests/test_docs_publishing.py -k "not example" From 9d8cb07b62e2a5b428d384639e598f0e254e2e61 Mon Sep 17 00:00:00 2001 From: Adarsh Divakaran Date: Mon, 16 Nov 2026 12:19:27 +0530 Subject: [PATCH 3/4] fix: clear AsyncClient close state when the caller is cancelled Finish aclose() bookkeeping in a done callback, so a cancelled caller no longer leaves closed sessions referenced or an unretrieved task exception. Declare anyio in the test extra, since the async tests use its pytest plugin. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyproject.toml | 6 +++++- serpapi/http.py | 20 ++++++++------------ tests/test_async_client.py | 35 ++++++++++++++++++++++++++++++++++- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 86b85b2..9712069 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,11 @@ classifiers = [ [project.optional-dependencies] color = ["pygments"] async = ["httpx2>=2.15.1; python_version >= '3.11'"] -test = ["pytest", "httpx2>=2.15.1; python_version >= '3.11'"] +test = [ + "pytest", + "httpx2>=2.15.1; python_version >= '3.11'", + "anyio; python_version >= '3.11'", +] docs = [ "sphinx>=8.2,<9; python_version >= '3.11'", "myst-parser>=4,<5; python_version >= '3.11'", diff --git a/serpapi/http.py b/serpapi/http.py index 66be1ef..4466739 100644 --- a/serpapi/http.py +++ b/serpapi/http.py @@ -231,21 +231,17 @@ async def aclose(self): self._closed = True if self._close_task is None: + # Close in a task, so that the sessions still close if the caller is cancelled. self._close_task = asyncio.create_task(self._close_sessions()) + self._close_task.add_done_callback(self._after_close) - close_task = self._close_task - try: - await asyncio.shield(close_task) - except asyncio.CancelledError: - if close_task.done(): - self._close_task = None - raise - except Exception: - self._close_task = None - raise - else: + await asyncio.shield(self._close_task) + + def _after_close(self, close_task): + self._close_task = None + # Reading the exception also stops asyncio reporting it as never retrieved. + if not close_task.cancelled() and close_task.exception() is None: self._option_sessions.clear() - self._close_task = None async def _close_sessions(self): sessions = [self.session, *self._option_sessions.values()] diff --git a/tests/test_async_client.py b/tests/test_async_client.py index 892a25b..960576d 100644 --- a/tests/test_async_client.py +++ b/tests/test_async_client.py @@ -472,14 +472,47 @@ async def delayed_close(): with pytest.raises(asyncio.CancelledError): await close_caller + # The close carries on in the background, without another aclose() call. + background_close = client._close_task allow_close.set() - await client.aclose() + await background_close + assert all(session.is_closed for session in sessions) + assert client._close_task is None + assert not client._option_sessions finally: allow_close.set() await client.aclose() +async def test_aclose_can_be_retried_after_the_background_close_fails(server, monkeypatch): + client = serpapi.AsyncClient(api_key="test-api-key") + close_started = asyncio.Event() + allow_close = asyncio.Event() + original_aclose = client.session.aclose + + async def failing_close(): + close_started.set() + await allow_close.wait() + raise OSError("close failed") + + monkeypatch.setattr(client.session, "aclose", failing_close) + close_caller = asyncio.create_task(client.aclose()) + await close_started.wait() + close_caller.cancel() + with pytest.raises(asyncio.CancelledError): + await close_caller + + background_close = client._close_task + allow_close.set() + await asyncio.wait({background_close}) + + monkeypatch.setattr(client.session, "aclose", original_aclose) + await client.aclose() + + assert client.session.is_closed + + async def test_closed_client_does_not_create_new_sessions(async_client): await async_client.aclose() From 28ffe92664a8c9f83f0539ff1fba9ee4b8dd9071 Mon Sep 17 00:00:00 2001 From: Adarsh Divakaran Date: Mon, 16 Nov 2026 14:58:31 +0530 Subject: [PATCH 4/4] docs: say AsyncClient ignores the stream option The AsyncClient method docstrings said stream is passed to the underlying HTTP request. The async transport discards it, as the user guide already states. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- serpapi/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/serpapi/core.py b/serpapi/core.py index 1fe9ba3..ecf1e58 100644 --- a/serpapi/core.py +++ b/serpapi/core.py @@ -273,7 +273,7 @@ async def search(self, params: dict = None, **kwargs): parameters as :meth:`Client.search `. :param params: Optional mapping of SerpApi search parameters such as ``engine``, ``q``, ``location``, and ``output``. - :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + :param kwargs: Additional SerpApi parameters or request options. ``timeout``, ``proxies``, ``verify``, and ``cert`` are applied to the underlying HTTP request. ``stream`` is accepted but ignored. **Learn more**: https://serpapi.com/search-api """ @@ -306,7 +306,7 @@ async def search_archive(self, params: dict = None, **kwargs): """Get a result from the SerpApi Search Archive API. Use it with ``await``. :param params: Archive parameters. Must include ``search_id``. ``output`` accepts ``json`` (default), ``html``, or ``md``. - :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + :param kwargs: Additional archive parameters or request options. ``timeout``, ``proxies``, ``verify``, and ``cert`` are applied to the underlying HTTP request. ``stream`` is accepted but ignored. **Learn more**: https://serpapi.com/search-archive-api """ @@ -378,7 +378,7 @@ async def locations(self, params: dict = None, **kwargs): """Get a list of supported Google locations. Use it with ``await``. :param params: Location API parameters such as ``q`` and ``limit``. - :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + :param kwargs: Additional location parameters or request options. ``timeout``, ``proxies``, ``verify``, and ``cert`` are applied to the underlying HTTP request. ``stream`` is accepted but ignored. **Learn more**: https://serpapi.com/locations-api """ @@ -403,7 +403,7 @@ async def account(self, params: dict = None, **kwargs): """Get SerpApi account information. Use it with ``await``. :param params: Account API parameters. - :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, ``stream``, and ``cert`` are passed to the underlying HTTP request. + :param kwargs: Additional account parameters or request options. ``timeout``, ``proxies``, ``verify``, and ``cert`` are applied to the underlying HTTP request. ``stream`` is accepted but ignored. **Learn more**: https://serpapi.com/account-api """