Add initial smoketests

Author: alb-rl

.github/workflows/ci.yml

@@ -95,4 +95,6 @@ jobs:
         run: ./scripts/bootstrap
 
       - name: Run tests
-        run: ./scripts/test
+        env:
+          RUN_SMOKETESTS: '0'
+        run: ./scripts/test --ignore=tests/smoketests

.github/workflows/smoketests.yml

@@ -0,0 +1,55 @@
+name: Smoketests
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Target environment"
+        type: choice
+        default: dev
+        options:
+          - dev
+          - prod
+
+jobs:
+  smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    defaults:
+      run:
+        working-directory: api-client-python
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          uv pip install -r requirements-dev.lock
+
+      - name: Configure environment
+        env:
+          DEV_KEY: ${{ secrets.RUNLOOP_SMOKETEST_DEV_API_KEY }}
+          PROD_KEY: ${{ secrets.RUNLOOP_SMOKETEST_PROD_API_KEY }}
+        run: |
+          if [ "${{ github.event.inputs.environment }}" = "prod" ]; then
+            echo "RUNLOOP_API_KEY=${PROD_KEY}" >> $GITHUB_ENV
+            echo "RUNLOOP_BASE_URL=https://api.runloop.ai" >> $GITHUB_ENV
+          else
+            echo "RUNLOOP_API_KEY=${DEV_KEY}" >> $GITHUB_ENV
+            echo "RUNLOOP_BASE_URL=https://api.runloop.pro" >> $GITHUB_ENV
+          fi
+          echo "DEBUG=false" >> $GITHUB_ENV
+          echo "RUN_SMOKETESTS=1" >> $GITHUB_ENV
+          echo "PYTHONPATH=${{ github.workspace }}/api-client-python/src" >> $GITHUB_ENV
+
+      - name: Run smoke tests (pytest via uv)
+        env:
+          # Force sequential to avoid overloading remote resources
+          PYTEST_ADDOPTS: "-n 1 -m smoketest"
+        run: |
+          uv run pytest -q -vv tests/smoketests

.gitignore

@@ -13,3 +13,5 @@ dist
 .envrc
 codegen.log
 Brewfile.lock.json
+
+.DS_Store

pyproject.toml

@@ -51,6 +51,7 @@ dev-dependencies = [
     "respx",
     "pytest",
     "pytest-asyncio",
+    "pytest-timeout",
     "ruff",
     "time-machine",
     "nox",
@@ -134,15 +135,17 @@ replacement = '[\1](https://github.com/runloopai/api-client-python/tree/main/\g<
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-addopts = "--tb=short -n auto"
+addopts = "--tb=short -n auto --dist=loadfile -m 'not smoketest'"
 xfail_strict = true
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 filterwarnings = [
   "error"
 ]
 markers = [
-  "skip_if_strict: skip test if using strict validation (for prism mock server issues)"
+  "skip_if_strict: skip test if using strict validation (for prism mock server issues)",
+  "timeout: per-test timeout provided by pytest-timeout",
+  "smoketest: end-to-end smoke tests against real API",
 ]
 
 [tool.pyright]

requirements-dev.lock

@@ -98,6 +98,8 @@ pyright==1.1.399
 pytest==8.3.3
     # via pytest-asyncio
     # via pytest-xdist
+pytest-timeout==2.3.1
+    # via runloop-api-client (dev)
 pytest-asyncio==0.24.0
 pytest-xdist==3.7.0
 python-dateutil==2.8.2

scripts/test

@@ -55,7 +55,15 @@ fi
 export DEFER_PYDANTIC_BUILD=false
 
 echo "==> Running tests"
-rye run pytest "$@"
+
+# By default, exclude smoketests unless explicitly enabled.
+# This ensures PR CI doesn't run E2E smoketests unless RUN_SMOKETESTS=1.
+PYTEST_ARGS=()
+if [ "${RUN_SMOKETESTS}" != "1" ]; then
+  PYTEST_ARGS+=( -m "not smoketest" )
+fi
+
+rye run pytest "${PYTEST_ARGS[@]}" "$@"
 
 echo "==> Running Pydantic v1 tests"
-rye run nox -s test-pydantic-v1 -- "$@"
+rye run nox -s test-pydantic-v1 -- "${PYTEST_ARGS[@]}" "$@"

tests/conftest.py

@@ -71,6 +71,16 @@ def pytest_collection_modifyitems(items: list[pytest.Function]) -> None:
             )
             item.add_marker(pytest.mark.skip(reason=reason))
 
+    # Finally, deselect smoketests by default unless explicitly enabled.
+    # This ensures PR CI will not run smoketests unintentionally.
+    if os.getenv("RUN_SMOKETESTS") != "1":
+        kept: list[pytest.Function] = []
+        for item in items:
+            if item.get_closest_marker("smoketest") is None:
+                kept.append(item)
+        if len(kept) != len(items):
+            items[:] = kept
+
 
 base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
 

tests/smoketests/README.md

@@ -0,0 +1,27 @@
+# Smoke tests
+
+End-to-end smoke tests run against the real API to validate critical flows (devboxes, snapshots, blueprints, executions/log tailing, scenarios/benchmarks).
+
+- Local run (requires `RUNLOOP_API_KEY`):
+
+```bash
+export RUNLOOP_API_KEY=...  # required
+# optionally override API base
+# export RUNLOOP_BASE_URL=https://api.runloop.ai
+
+# Install deps and run via uv
+uv pip install -r requirements-dev.lock
+
+# Run all tests
+RUN_SMOKETESTS=1 uv run pytest -q -vv tests/smoketests
+
+# Run a single file
+RUN_SMOKETESTS=1 uv run pytest -q -vv tests/smoketests/test_devboxes.py
+
+# Run a single test by name
+RUN_SMOKETESTS=1 uv run pytest -q -k "test_create_and_await_running_timeout" tests/smoketests/test_devboxes.py
+```
+
+- GitHub Actions: add repo secret `RUNLOOP_SMOKETEST_DEV_API_KEY` and `RUNLOOP_SMOKETEST_PROD_API_KEY`. The workflow `.github/workflows/smoketests.yml` supports an input `environment` (dev|prod) and runs these tests in CI.
+
+

tests/smoketests/__init__.py

@@ -0,0 +1,6 @@
+"""Test package for smoketests.
+
+Ensures relative imports like `from .utils import ...` work under pytest.
+"""
+
+

tests/smoketests/test_blueprints.py

@@ -0,0 +1,61 @@
+import pytest
+
+from runloop_api_client.lib.polling import PollingConfig
+
+from .utils import make_client, unique_name
+
+pytestmark = [pytest.mark.smoketest]
+
+
+client = make_client()
+
+
+"""
+Tests are run sequentially and can be dependent on each other. 
+This is to avoid overloading resources and save efficiency.
+"""
+_blueprint_id = None
+_blueprint_name = unique_name("bp")
+
+
+def teardown_module() -> None:
+    global _blueprint_id
+    if _blueprint_id:
+        try:
+            client.blueprints.delete(_blueprint_id)
+        except Exception:
+            pass
+
+
+@pytest.mark.timeout(30)
+def test_create_blueprint_and_await_build() -> None:
+    global _blueprint_id
+    created = client.blueprints.create_and_await_build_complete(
+        name=_blueprint_name,
+        polling_config=PollingConfig(max_attempts=180, interval_seconds=5.0, timeout_seconds=30 * 60),
+    )
+    assert created.status == "build_complete"
+    _blueprint_id = created.id
+
+
+@pytest.mark.timeout(30)
+def test_start_devbox_from_base_blueprint_by_id() -> None:
+    assert _blueprint_id
+    devbox = client.devboxes.create_and_await_running(
+        blueprint_id=_blueprint_id,
+        polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
+    )
+    assert devbox.blueprint_id == _blueprint_id
+    assert devbox.status == "running"
+    client.devboxes.shutdown(devbox.id)
+
+
+@pytest.mark.timeout(30)
+def test_start_devbox_from_base_blueprint_by_name() -> None:
+    devbox = client.devboxes.create_and_await_running(
+        blueprint_name=_blueprint_name,
+        polling_config=PollingConfig(max_attempts=120, interval_seconds=5.0, timeout_seconds=20 * 60),
+    )
+    assert devbox.blueprint_id
+    assert devbox.status == "running"
+    client.devboxes.shutdown(devbox.id)