Add benchmarking script

Author: MarkusSintonen

requirements.txt

@@ -21,3 +21,8 @@ pytest==8.2.0
 pytest-httpbin==2.0.0
 pytest-trio==0.8.0
 werkzeug<2.1  # See: https://github.com/psf/httpbin/issues/35
+
+# Benchmarking and profiling
+aiohttp==3.9.5
+matplotlib==3.9.0
+pyinstrument==4.6.2

scripts/benchmark

@@ -0,0 +1,13 @@
+#!/bin/sh -e
+
+export PREFIX=""
+if [ -d 'venv' ] ; then
+    export PREFIX="venv/bin/"
+fi
+
+set -x
+
+${PREFIX}python tests/benchmark/server.py &
+SERVER_PID=$!
+${PREFIX}python tests/benchmark/client.py
+kill $SERVER_PID

tests/benchmark/client.py

@@ -0,0 +1,113 @@
+import asyncio
+import os
+import time
+from contextlib import contextmanager
+from typing import Any, Coroutine, Iterator
+
+import aiohttp
+import matplotlib.pyplot as plt
+import pyinstrument
+from matplotlib.axes import Axes
+
+import httpcore
+
+PORT = 1234
+URL = f"http://localhost:{PORT}/req"
+REPEATS = 10
+REQUESTS = 500
+CONCURRENCY = 20
+POOL_LIMIT = 100
+PROFILE = False
+os.environ["HTTPCORE_PREFER_ANYIO"] = "0"
+
+
+def duration(start: float) -> int:
+    return int((time.monotonic() - start) * 1000)
+
+
+@contextmanager
+def profile():
+    if not PROFILE:
+        yield
+        return
+    with pyinstrument.Profiler() as profiler:
+        yield
+    profiler.open_in_browser()
+
+
+async def gather_limited_concurrency(
+    coros: Iterator[Coroutine[Any, Any, Any]], concurrency: int = CONCURRENCY
+) -> None:
+    sem = asyncio.Semaphore(concurrency)
+
+    async def coro_with_sem(coro: Coroutine[Any, Any, Any]) -> None:
+        async with sem:
+            await coro
+
+    await asyncio.gather(*(coro_with_sem(c) for c in coros))
+
+
+async def run_requests(axis: Axes) -> None:
+    async def httpcore_get(
+        pool: httpcore.AsyncConnectionPool, timings: list[int]
+    ) -> None:
+        start = time.monotonic()
+        res = await pool.request("GET", URL)
+        assert len(await res.aread()) == 2000
+        assert res.status == 200, f"status_code={res.status}"
+        timings.append(duration(start))
+
+    async def aiohttp_get(session: aiohttp.ClientSession, timings: list[int]) -> None:
+        start = time.monotonic()
+        async with session.request("GET", URL) as res:
+            assert len(await res.read()) == 2000
+            assert res.status == 200, f"status={res.status}"
+        timings.append(duration(start))
+
+    async with httpcore.AsyncConnectionPool(max_connections=POOL_LIMIT) as pool:
+        # warmup
+        await gather_limited_concurrency(
+            (httpcore_get(pool, []) for _ in range(REQUESTS)), CONCURRENCY * 2
+        )
+
+        timings: list[int] = []
+        start = time.monotonic()
+        with profile():
+            for _ in range(REPEATS):
+                await gather_limited_concurrency(
+                    (httpcore_get(pool, timings) for _ in range(REQUESTS))
+                )
+        axis.plot(
+            [*range(len(timings))], timings, label=f"httpcore (tot={duration(start)}ms)"
+        )
+
+    connector = aiohttp.TCPConnector(limit=POOL_LIMIT)
+    async with aiohttp.ClientSession(connector=connector) as session:
+        # warmup
+        await gather_limited_concurrency(
+            (aiohttp_get(session, []) for _ in range(REQUESTS)), CONCURRENCY * 2
+        )
+
+        timings = []
+        start = time.monotonic()
+        for _ in range(REPEATS):
+            await gather_limited_concurrency(
+                (aiohttp_get(session, timings) for _ in range(REQUESTS))
+            )
+        axis.plot(
+            [*range(len(timings))], timings, label=f"aiohttp (tot={duration(start)}ms)"
+        )
+
+
+def main() -> None:
+    fig, ax = plt.subplots()
+    asyncio.run(run_requests(ax))
+    plt.legend(loc="upper left")
+    ax.set_xlabel("# request")
+    ax.set_ylabel("[ms]")
+    plt.show()
+    print("DONE", flush=True)
+
+
+if __name__ == "__main__":
+    main()

tests/benchmark/server.py

@@ -0,0 +1,22 @@
+import asyncio
+
+from aiohttp import web
+
+PORT = 1234
+RESP = "a" * 2000
+SLEEP = 0.01
+
+
+async def handle(_request):
+    await asyncio.sleep(SLEEP)
+    return web.Response(text=RESP)
+
+
+def main() -> None:
+    app = web.Application()
+    app.add_routes([web.get("/req", handle)])
+    web.run_app(app, host="localhost", port=PORT)
+
+
+if __name__ == "__main__":
+    main()