Parameterize smoketests to run both sync and async tests

Author: alb-rl

src/runloop_api_client/resources/devboxes/devboxes.py

@@ -6,7 +6,9 @@
 from typing_extensions import Literal
 
 import httpx
-from uuid_utils import uuid7
+
+# uuid_utils is not typed
+from uuid_utils import uuid7  # type: ignore
 
 from .lsp import (
     LspResource,
@@ -809,7 +811,7 @@ def execute_and_await_completion(
         return the result within the initial request's timeout. If the execution is not yet
         complete, it switches to using wait_for_command to minimize latency while waiting.
         """
-        command_id = str(uuid7())
+        command_id = str(uuid7()) # type: ignore
         execution = self.execute(
             devbox_id,
             command=command,
@@ -2251,7 +2253,7 @@ async def execute_and_await_completion(
         complete, it switches to using wait_for_command to minimize latency while waiting.
         """
 
-        command_id = str(uuid7())
+        command_id = str(uuid7()) # type: ignore
         execution = await self.execute(
             devbox_id,
             command=command,

tests/smoketests/README.md

@@ -1,6 +1,7 @@
 # Smoke tests
 
-End-to-end smoke tests run against the real API to validate critical flows (devboxes, snapshots, blueprints, executions/log tailing, scenarios/benchmarks).
+End-to-end smoke tests run against the real API to validate critical flows (devboxes, snapshots, blueprints, executions/log tailing, scenarios/benchmarks). Theses smoketests run both the 
+async and sync clients.
 
 - Local run (requires `RUNLOOP_API_KEY`):
 
@@ -13,13 +14,13 @@ export RUNLOOP_API_KEY=...  # required
 uv pip install -r requirements-dev.lock
 
 # Run all tests
-RUN_SMOKETESTS=1 uv run pytest -q -vv tests/smoketests
+RUN_SMOKETESTS=1 uv run pytest -q -vv -m smoketest tests/smoketests
 
 # Run a single file
-RUN_SMOKETESTS=1 uv run pytest -q -vv tests/smoketests/test_devboxes.py
+RUN_SMOKETESTS=1 uv run pytest -q -vv -m smoketest tests/smoketests/test_devboxes.py
 
 # Run a single test by name
-RUN_SMOKETESTS=1 uv run pytest -q -k "test_create_and_await_running_timeout" tests/smoketests/test_devboxes.py
+RUN_SMOKETESTS=1 uv run pytest -q -k -m smoketest "test_create_and_await_running_timeout" tests/smoketests/test_devboxes.py
 ```
 
 - GitHub Actions: add repo secret `RUNLOOP_SMOKETEST_DEV_API_KEY` and `RUNLOOP_SMOKETEST_PROD_API_KEY`. The workflow `.github/workflows/smoketests.yml` supports an input `environment` (dev|prod) and runs these tests in CI.

tests/smoketests/conftest.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Iterator
+
+import pytest
+from _pytest.fixtures import FixtureRequest  # pyright: ignore[reportPrivateImportUsage]
+
+from runloop_api_client import Runloop
+
+from .utils import make_client, make_async_client_adapter
+
+"""
+This file is used to create a client fixture for the tests.
+It makes it possible to run the tests with both sync and async clients.
+"""
+
+
+@pytest.fixture(scope="module", params=["sync", "async"], ids=["sync-client", "async-client"])
+def client(request: FixtureRequest) -> Iterator[Runloop]:
+    if request.param == "sync":
+        c: Runloop = make_client()
+        try:
+            yield c
+        finally:
+            try:
+                # Runloop supports close()
+                c.close()
+            except Exception:
+                pass
+    else:
+        c: Runloop = make_async_client_adapter()
+        try:
+            yield c
+        finally:
+            try:
+                c.close()
+            except Exception:
+                pass

tests/smoketests/test_blueprints.py

@@ -1,13 +1,26 @@
+from __future__ import annotations
+
+from typing import Iterator
+
 import pytest
 
+from runloop_api_client import Runloop
 from runloop_api_client.lib.polling import PollingConfig
 
-from .utils import make_client, unique_name
+from .utils import unique_name
 
 pytestmark = [pytest.mark.smoketest]
 
 
-client = make_client()
+@pytest.fixture(autouse=True, scope="module")
+def _cleanup(client: Runloop) -> Iterator[None]:  # pyright: ignore[reportUnusedFunction]
+    yield
+    global _blueprint_id
+    if _blueprint_id:
+        try:
+            client.blueprints.delete(_blueprint_id)
+        except Exception:
+            pass
 
 
 """
@@ -18,17 +31,8 @@
 _blueprint_name = unique_name("bp")
 
 
-def teardown_module() -> None:
-    global _blueprint_id
-    if _blueprint_id:
-        try:
-            client.blueprints.delete(_blueprint_id)
-        except Exception:
-            pass
-
-
 @pytest.mark.timeout(30)
-def test_create_blueprint_and_await_build() -> None:
+def test_create_blueprint_and_await_build(client: Runloop) -> None:
     global _blueprint_id
     created = client.blueprints.create_and_await_build_complete(
         name=_blueprint_name,
@@ -39,7 +43,7 @@ def test_create_blueprint_and_await_build() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_start_devbox_from_base_blueprint_by_id() -> None:
+def test_start_devbox_from_base_blueprint_by_id(client: Runloop) -> None:
     assert _blueprint_id
     devbox = client.devboxes.create_and_await_running(
         blueprint_id=_blueprint_id,
@@ -51,7 +55,7 @@ def test_start_devbox_from_base_blueprint_by_id() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_start_devbox_from_base_blueprint_by_name() -> None:
+def test_start_devbox_from_base_blueprint_by_name(client: Runloop) -> None:
     devbox = client.devboxes.create_and_await_running(
         blueprint_name=_blueprint_name,
         polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),

tests/smoketests/test_devboxes.py

@@ -1,13 +1,27 @@
+from __future__ import annotations
+
+from typing import Iterator
+
 import pytest
 
+from runloop_api_client import Runloop
 from runloop_api_client.lib.polling import PollingConfig, PollingTimeout
 
-from .utils import make_client, unique_name
+from .utils import unique_name
 
 pytestmark = [pytest.mark.smoketest]
 
 
-client = make_client()
+@pytest.fixture(autouse=True, scope="module")
+def _cleanup(client: Runloop) -> Iterator[None]:  # pyright: ignore[reportUnusedFunction]
+    yield
+    global _devbox_id
+    if _devbox_id:
+        try:
+            client.devboxes.shutdown(_devbox_id)
+        except Exception:
+            pass
+
 
 """
 Tests are run sequentially and can be dependent on each other. 
@@ -17,14 +31,14 @@
 
 
 @pytest.mark.timeout(30)
-def test_create_devbox() -> None:
+def test_create_devbox(client: Runloop) -> None:
     created = client.devboxes.create(name=unique_name("smoke-devbox"))
     assert created.id
     client.devboxes.shutdown(created.id)
 
 
 @pytest.mark.timeout(30)
-def test_await_running_create_and_await_running() -> None:
+def test_await_running_create_and_await_running(client: Runloop) -> None:
     global _devbox_id
     created = client.devboxes.create_and_await_running(
         name=unique_name("smoketest-devbox2"),
@@ -34,27 +48,27 @@ def test_await_running_create_and_await_running() -> None:
     _devbox_id = created.id
 
 
-def test_list_devboxes() -> None:
+def test_list_devboxes(client: Runloop) -> None:
     page = client.devboxes.list(limit=10)
     assert isinstance(page.devboxes, list)
     assert len(page.devboxes) > 0
 
 
-def test_retrieve_devbox() -> None:
+def test_retrieve_devbox(client: Runloop) -> None:
     assert _devbox_id
     view = client.devboxes.retrieve(_devbox_id)
     assert view.id == _devbox_id
 
 
-def test_shutdown_devbox() -> None:
+def test_shutdown_devbox(client: Runloop) -> None:
     assert _devbox_id
     view = client.devboxes.shutdown(_devbox_id)
     assert view.id == _devbox_id
     assert view.status == "shutdown"
 
 
 @pytest.mark.timeout(90)
-def test_create_and_await_running_long_set_up() -> None:
+def test_create_and_await_running_long_set_up(client: Runloop) -> None:
     created = client.devboxes.create_and_await_running(
         name=unique_name("smoketest-devbox-await-running-long-set-up"),
         launch_parameters={"launch_commands": ["sleep 70"]},
@@ -65,7 +79,7 @@ def test_create_and_await_running_long_set_up() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_create_and_await_running_timeout() -> None:
+def test_create_and_await_running_timeout(client: Runloop) -> None:
     with pytest.raises(PollingTimeout):
         client.devboxes.create_and_await_running(
             name=unique_name("smoketest-devbox-await-running-timeout"),

tests/smoketests/test_executions.py

@@ -1,13 +1,26 @@
+from __future__ import annotations
+
+from typing import Iterator
+
 import pytest
 
+from runloop_api_client import Runloop
 from runloop_api_client.lib.polling import PollingConfig
 
-from .utils import make_client, unique_name
+from .utils import unique_name
 
 pytestmark = [pytest.mark.smoketest]
 
 
-client = make_client()
+@pytest.fixture(autouse=True, scope="module")
+def _cleanup(client: Runloop) -> Iterator[None]:  # pyright: ignore[reportUnusedFunction]
+    yield
+    global _devbox_id
+    if _devbox_id:
+        try:
+            client.devboxes.shutdown(_devbox_id)
+        except Exception:
+            pass
 
 
 """
@@ -18,17 +31,8 @@
 _exec_id = None
 
 
-@pytest.fixture(scope="session")
-def some_function_name():
-    # setup
-    yield
-    # teardown
-    if _devbox_id:
-        client.devboxes.shutdown(_devbox_id)
-
-
 @pytest.mark.timeout(30)
-def test_launch_devbox() -> None:
+def test_launch_devbox(client: Runloop) -> None:
     global _devbox_id
     created = client.devboxes.create_and_await_running(
         name=unique_name("exec-devbox"),
@@ -38,7 +42,7 @@ def test_launch_devbox() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_execute_async_and_await_completion() -> None:
+def test_execute_async_and_await_completion(client: Runloop) -> None:
     assert _devbox_id
     global _exec_id
     started = client.devboxes.executions.execute_async(_devbox_id, command="echo hello && sleep 1")
@@ -52,7 +56,7 @@ def test_execute_async_and_await_completion() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_tail_stdout_logs() -> None:
+def test_tail_stdout_logs(client: Runloop) -> None:
     assert _devbox_id and _exec_id
     stream = client.devboxes.executions.stream_stdout_updates(execution_id=_exec_id, devbox_id=_devbox_id)
     received = ""
@@ -64,7 +68,7 @@ def test_tail_stdout_logs() -> None:
 
 
 @pytest.mark.timeout(30)
-def test_execute_and_await_completion() -> None:
+def test_execute_and_await_completion(client: Runloop) -> None:
     assert _devbox_id
     completed = client.devboxes.execute_and_await_completion(
         _devbox_id,
@@ -75,7 +79,7 @@ def test_execute_and_await_completion() -> None:
 
 
 @pytest.mark.timeout(90)
-def test_execute_and_await_completion_long_running() -> None:
+def test_execute_and_await_completion_long_running(client: Runloop) -> None:
     assert _devbox_id
     completed = client.devboxes.execute_and_await_completion(
         _devbox_id,