Various updates (#24)

Author: provinzkraut

Diff for: .pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     hooks:
       - id: ruff
         args: ["--fix"]
-        exclude: "frameworks*"
+        exclude: "test_data"
   - repo: https://github.com/psf/black
     rev: 23.7.0
     hooks:

Diff for: README.md

@@ -23,6 +23,7 @@
 > [**_Starlite has been renamed to Litestar_**](https://litestar.dev/about/organization.html#litestar-and-starlite)
 
 This is an API performance test comparing:
+
 1. [Litestar](https://github.com/litestar-org/litestar)
 2. [Starlite v1.5x](https://github.com/litestar-org/litestar/tree/v1.51)
 3. [Starlette](https://github.com/encode/starlette)
@@ -32,15 +33,6 @@ This is an API performance test comparing:
 
 Using the [bombardier](https://github.com/codesenberg/bombardier) HTTP benchmarking tool.
 
-## Last Run Results
-
-![Plain Text Results](result.png)
-
-You can view the last run results under the `/results` folder - it contains json files with the output.
-The plotting is done using pandas - script is under `/analysis`.
-
-Note: PRs improving the analysis script are welcome.
-
 ## Test Setup
 
 Setup is identical for all frameworks.
@@ -153,7 +145,7 @@ You can run a single test by specifying its full name and category:
 #### Test Settings
 
 |                                                                                                                                                                       |                                                                 |
-|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- |
 | -r, --rebuild                                                                                                                                                         | rebuild docker images                                           |
 | -L, --latency                                                                                                                                                         | run latency tests                                               |
 | -R, --rps                                                                                                                                                             | run RPS tests                                                   |

Diff for: asgi_bench/build.py

@@ -37,7 +37,7 @@ def build_docker_images(framework_specs: list[FrameworkSpec], rebuild: bool = Fa
     client = docker.from_env()
 
     with console.status("[yellow]Building runner image"):
-        client.images.build(path=".", dockerfile="DockerfileBench", tag="starlite-api-bench:runner")
+        client.images.build(path=".", dockerfile="DockerfileBench", tag="litestar-bench:runner")
     console.print("[green]  Runner image built successfully")
 
     for framework in framework_specs:
@@ -68,5 +68,5 @@ def remove_docker_images(client: docker.DockerClient | None = None, force: bool
         client = docker.from_env()
     with console.status("[yellow]Removing all images"):
         for image in client.images.list():
-            if any(tag.startswith("starlite-api-bench:") for tag in image.tags):
+            if any(tag.startswith("litestar-bench:") for tag in image.tags):
                 image.remove(force=force)

Diff for: asgi_bench/runner.py

@@ -139,7 +139,7 @@ def _run_image(self, image: str) -> Container:
 
     def _run_bench_in_container(self, *args: str) -> str:
         container = self.docker_client.containers.run(
-            "starlite-api-bench:runner", "./bombardier " + " ".join(args), network_mode="host", detach=True
+            "litestar-bench:runner", "./bombardier " + " ".join(args), network_mode="host", detach=True
         )
         container.wait()
         return container.logs().decode()
@@ -162,7 +162,7 @@ def _validate_bench_endpoint(self, test_spec: TestSpec) -> bool:
     def _stop_all_containers(self) -> None:
         with self.console.status("[yellow]Stopping running containers"):
             for container in self.docker_client.containers.list(ignore_removed=True):
-                if any(tag.startswith("starlite-api-bench:") for tag in container.image.tags):
+                if any(tag.startswith("litestar-bench:") for tag in container.image.tags):
                     container.kill()
 
     def run_benchmark(self, test_spec: TestSpec) -> dict[str, Any]:
@@ -204,10 +204,7 @@ def provide_service(self, spec: FrameworkSpec) -> Generator[bool, None, None]:
             try:
                 container.kill()
             except APIError as error:
-                if (
-                    error.status_code != 409
-                    or "not running" not in error.explanation
-                ):
+                if not (error.status_code == 409 and "not running" in error.explanation):
                     # the container stopped for reasons
                     raise error
             yield False

Diff for: asgi_bench/spec.py

@@ -37,7 +37,15 @@ def supports_framework(self, framework: str, mode: EndpointMode) -> bool:
 class TestCategory:
     name: EndpointCategory
     endpoints: list[Endpoint]
-    frameworks: tuple[Framework, ...] = ("starlite", "starlette", "fastapi", "sanic", "blacksheep", "quart")
+    frameworks: tuple[Framework, ...] = (
+        "litestar",
+        "starlite",
+        "starlette",
+        "fastapi",
+        "sanic",
+        "blacksheep",
+        "quart",
+    )
 
 
 TEST_CATEGORIES: list[TestCategory] = [
@@ -91,7 +99,7 @@ class TestCategory:
     ),
     TestCategory(
         name="dependency-injection",
-        frameworks=("starlite", "fastapi", "sanic", "blacksheep"),
+        frameworks=("litestar", "fastapi", "sanic", "blacksheep"),
         endpoints=[
             Endpoint(path="dependencies-sync", name="dependencies sync"),
             Endpoint(path="dependencies-async", name="dependencies async", exclude=["sanic", "blacksheep"]),
@@ -100,7 +108,7 @@ class TestCategory:
     ),
     TestCategory(
         name="serialization",
-        frameworks=("starlite", "fastapi"),
+        frameworks=("litestar", "fastapi"),
         endpoints=[
             Endpoint(path="serialize-pydantic-50", name="serialize pydantic, 50 objects"),
             Endpoint(path="serialize-pydantic-100", name="serialize pydantic, 100 objects"),
@@ -254,7 +262,7 @@ def make_spec(
 
 
 FRAMEWORK_REPOS: dict[Framework, str] = {
-    "starlite": "https://github.com/starlite-api/starlite.git",
+    "litestar": "https://github.com/litestar-org/litestar.git",
     "starlette": "https://github.com/encode/starlette.git",
     "fastapi": "https://github.com/tiangolo/fastapi.git",
     "sanic": "https://github.com/sanic-org/sanic.git",

Diff for: asgi_bench/types.py

@@ -17,7 +17,7 @@
 ]
 BenchmarkMode = Literal["rps", "latency"]
 VersionPrefix = Literal["pip", "git", "docker", "file"]
-Framework = Literal["starlite", "starlette", "fastapi", "sanic", "blacksheep", "quart", "litestar"]
+Framework = Literal["litestar", "starlite", "starlette", "fastapi", "sanic", "blacksheep", "quart"]
 
 
 @dataclass
@@ -85,7 +85,7 @@ def is_pip_target(self) -> bool:
     @property
     def image_tag(self) -> str:
         versioned_name = self.version_name.replace(":", "_").replace("/", "_").replace(".git", "")
-        return f"starlite-api-bench:{versioned_name}"
+        return f"litestar-bench:{versioned_name}"
 
     @property
     def build_stage_image(self) -> str | None:
@@ -116,7 +116,9 @@ def pip_package(self) -> str:
             from .spec import FRAMEWORK_REPOS
 
             return f"git+{FRAMEWORK_REPOS[self.name]}@{version}"
-        return version if prefix == "file" else f"{self.name}=={self.version}"
+        if prefix == "file":
+            return version
+        return f"{self.name}=={self.version}"
 
     @property
     def extra_requirements(self) -> list[str]:

Diff for: frameworks/__init__.py

@@ -1,3 +1,19 @@
-from . import blacksheep_app, fastapi_app, sanic_app, starlette_app, starlite_app, quart_app
+from . import (
+    blacksheep_app,
+    fastapi_app,
+    litestar_app,
+    quart_app,
+    sanic_app,
+    starlette_app,
+    starlite_app,
+)
 
-__all__ = ["blacksheep_app", "fastapi_app", "sanic_app", "starlite_app", "starlette_app", "quart_app"]
+__all__ = [
+    "blacksheep_app",
+    "fastapi_app",
+    "litestar_app",
+    "quart_app",
+    "sanic_app",
+    "starlette_app",
+    "starlite_app",
+]

Diff for: frameworks/blacksheep_app.py

@@ -375,7 +375,7 @@ async def async_dependencies_sync(
 
 @app.router.post("/async-post-json")
 async def async_post_json(request: Request) -> Response:
-    data = await request.json()
+    await request.json()
     return no_content()
 
 
@@ -410,11 +410,11 @@ def sync_post_form_urlencoded(request: Request, data: FromForm) -> Response:
 
 @app.router.post("/sync-post-file")
 def sync_post_file(request: Request, data: FromFiles) -> Response:
-    content = data.value[0].data
+    data.value[0].data
     return no_content()
 
 
 @app.router.post("/async-post-file")
 async def async_post_file(request: Request, data: FromFiles) -> Response:
-    content = data.value[0].data
+    data.value[0].data
     return no_content()

Diff for: frameworks/fastapi_app.py

@@ -307,14 +307,14 @@ def sync_dependency_one() -> str:
 
 
 def sync_dependency_two(
-    injected_sync_one: str = Depends(sync_dependency_one),  # noqa: B008
+    injected_sync_one: str = Depends(sync_dependency_one),
 ) -> list[str]:
     time.sleep(0.00000001)
     return [injected_sync_one, "sync_dependency_two"]
 
 
 def sync_dependency_three(
-    injected_sync_two: list[str] = Depends(sync_dependency_two),  # noqa: B008
+    injected_sync_two: list[str] = Depends(sync_dependency_two),
 ) -> list[str]:
     time.sleep(0.00000001)
     return [*injected_sync_two, "sync_dependency_three"]
@@ -326,84 +326,84 @@ async def async_dependency_one() -> str:
 
 
 async def async_dependency_two(
-    injected_async_one: str = Depends(async_dependency_one),  # noqa: B008
+    injected_async_one: str = Depends(async_dependency_one),
 ) -> list[str]:
     await anyio.sleep(0.00000001)
     return [injected_async_one, "async_dependency_two"]
 
 
 async def async_dependency_three(
-    injected_async_two: list[str] = Depends(async_dependency_two),  # noqa: B008
+    injected_async_two: list[str] = Depends(async_dependency_two),
 ) -> list[str]:
     await anyio.sleep(0.00000001)
     return [*injected_async_two, "async_dependency_three"]
 
 
 async def dependencies_mixed(
-    injected_sync_three: list[str] = Depends(sync_dependency_three),  # noqa: B008
-    injected_async_three: list[str] = Depends(async_dependency_three),  # noqa: B008
+    injected_sync_three: list[str] = Depends(sync_dependency_three),
+    injected_async_three: list[str] = Depends(async_dependency_three),
 ) -> tuple[list[str], list[str]]:
     return injected_sync_three, injected_async_three
 
 
 @app.get("/sync-dependencies-sync")
 def sync_dependencies_sync(
-    injected_sync_one: str = Depends(sync_dependency_one),  # noqa: B008
-    injected_sync_two: list[str] = Depends(sync_dependency_two),  # noqa: B008
-    injected_sync_three: list[str] = Depends(sync_dependency_three),  # noqa: B008
+    injected_sync_one: str = Depends(sync_dependency_one),
+    injected_sync_two: list[str] = Depends(sync_dependency_two),
+    injected_sync_three: list[str] = Depends(sync_dependency_three),
 ) -> list[str]:
     return injected_sync_three
 
 
 @app.get("/sync-dependencies-async")
 def sync_dependencies_async(
-    injected_async_one: str = Depends(async_dependency_one),  # noqa: B008
-    injected_async_two: list[str] = Depends(async_dependency_two),  # noqa: B008
-    injected_async_three: list[str] = Depends(async_dependency_three),  # noqa: B008
+    injected_async_one: str = Depends(async_dependency_one),
+    injected_async_two: list[str] = Depends(async_dependency_two),
+    injected_async_three: list[str] = Depends(async_dependency_three),
 ) -> list[str]:
     return injected_async_three
 
 
 @app.get("/sync-dependencies-mixed")
 def sync_dependencies_mixed(
-    injected_sync_one: str = Depends(sync_dependency_one),  # noqa: B008
-    injected_sync_two: list[str] = Depends(sync_dependency_two),  # noqa: B008
-    injected_sync_three: list[str] = Depends(sync_dependency_three),  # noqa: B008
-    injected_async_one: str = Depends(async_dependency_one),  # noqa: B008
-    injected_async_two: list[str] = Depends(async_dependency_two),  # noqa: B008
-    injected_async_three: list[str] = Depends(async_dependency_three),  # noqa: B008
-    injected_mixed: tuple[list[str], list[str]] = Depends(dependencies_mixed),  # noqa: B008
+    injected_sync_one: str = Depends(sync_dependency_one),
+    injected_sync_two: list[str] = Depends(sync_dependency_two),
+    injected_sync_three: list[str] = Depends(sync_dependency_three),
+    injected_async_one: str = Depends(async_dependency_one),
+    injected_async_two: list[str] = Depends(async_dependency_two),
+    injected_async_three: list[str] = Depends(async_dependency_three),
+    injected_mixed: tuple[list[str], list[str]] = Depends(dependencies_mixed),
 ) -> tuple[list[str], list[str]]:
     return injected_mixed
 
 
 @app.get("/async-dependencies-sync")
 async def async_dependencies_sync(
-    injected_sync_one: str = Depends(sync_dependency_one),  # noqa: B008
-    injected_sync_two: list[str] = Depends(sync_dependency_two),  # noqa: B008
-    injected_sync_three: list[str] = Depends(sync_dependency_three),  # noqa: B008
+    injected_sync_one: str = Depends(sync_dependency_one),
+    injected_sync_two: list[str] = Depends(sync_dependency_two),
+    injected_sync_three: list[str] = Depends(sync_dependency_three),
 ) -> list[str]:
     return injected_sync_three
 
 
 @app.get("/async-dependencies-async")
 async def async_dependencies_async(
-    injected_async_one: str = Depends(async_dependency_one),  # noqa: B008
-    injected_async_two: list[str] = Depends(async_dependency_two),  # noqa: B008
-    injected_async_three: list[str] = Depends(async_dependency_three),  # noqa: B008
+    injected_async_one: str = Depends(async_dependency_one),
+    injected_async_two: list[str] = Depends(async_dependency_two),
+    injected_async_three: list[str] = Depends(async_dependency_three),
 ) -> list[str]:
     return injected_async_three
 
 
 @app.get("/async-dependencies-mixed")
 async def async_dependencies_mixed(
-    injected_sync_one: str = Depends(sync_dependency_one),  # noqa: B008
-    injected_sync_two: list[str] = Depends(sync_dependency_two),  # noqa: B008
-    injected_sync_three: list[str] = Depends(sync_dependency_three),  # noqa: B008
-    injected_async_one: str = Depends(async_dependency_one),  # noqa: B008
-    injected_async_two: list[str] = Depends(async_dependency_two),  # noqa: B008
-    injected_async_three: list[str] = Depends(async_dependency_three),  # noqa: B008
-    injected_mixed: tuple[list[str], list[str]] = Depends(dependencies_mixed),  # noqa: B008
+    injected_sync_one: str = Depends(sync_dependency_one),
+    injected_sync_two: list[str] = Depends(sync_dependency_two),
+    injected_sync_three: list[str] = Depends(sync_dependency_three),
+    injected_async_one: str = Depends(async_dependency_one),
+    injected_async_two: list[str] = Depends(async_dependency_two),
+    injected_async_three: list[str] = Depends(async_dependency_three),
+    injected_mixed: tuple[list[str], list[str]] = Depends(dependencies_mixed),
 ) -> tuple[list[str], list[str]]:
     return injected_mixed
 
@@ -492,25 +492,25 @@ async def async_post_json(data: list) -> None:
 
 @app.post("/async-post-multipart-form", status_code=HTTP_204_NO_CONTENT)
 async def async_post_multipart_form(request: Request) -> None:
-    data = await request.form()
+    await request.form()
 
 
 # form urlencoded
 
 
 @app.post("/async-post-form-urlencoded", status_code=HTTP_204_NO_CONTENT)
 async def async_post_form_urlencoded(request: Request) -> None:
-    data = await request.form()
+    await request.form()
 
 
 # upload files
 
 
 @app.post("/sync-post-file", status_code=HTTP_204_NO_CONTENT)
 def sync_post_file(test_file: UploadFile) -> None:
-    content = test_file.file.read()
+    test_file.file.read()
 
 
 @app.post("/async-post-file", status_code=HTTP_204_NO_CONTENT)
 async def async_post_file(test_file: UploadFile) -> None:
-    content = await test_file.read()
+    await test_file.read()