Split liveness and readiness health checks

devops/docker-compose.quantara.yaml

@@ -35,7 +35,7 @@ services:
     networks:
       - app_network
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"]
+      test: ["CMD-SHELL", "curl -f http://localhost:8000/readyz || exit 1"]
       interval: 30s
       timeout: 10s
       retries: 3

quantara/web_app/api/main.py

@@ -20,6 +20,7 @@
 from slowapi.middleware import SlowAPIMiddleware
 from sqlalchemy import text
 from sqlalchemy.orm import Session
+import aiohttp
 import redis.asyncio as redis
 
 from web_app.api.rate_limiter import limiter
@@ -189,35 +190,80 @@ async def request_id_middleware(request: Request, call_next):
 app.middleware("http")(protocol_pause_middleware)
 
 
-@app.get("/health", tags=["Health"], summary="Health check endpoint")
-async def health_check(response: Response, db: Session = Depends(get_database)):
-    """Returns 200 OK when the service is running and dependencies are healthy."""
-    health_status = {"status": "healthy", "database": "up", "redis": "up"}
-    is_healthy = True
-
-    # Check Database
+async def _check_database(db: Session) -> str:
     try:
-        # Use asyncio.to_thread for synchronous SQLAlchemy call to prevent blocking the event loop
         await asyncio.wait_for(
             asyncio.to_thread(db.execute, text("SELECT 1")), timeout=2.0
         )
+        return "up"
     except Exception as e:
-        logger.error("health_check_db_failed", error=str(e))
-        health_status["database"] = "down"
-        is_healthy = False
+        logger.error("readyz_db_failed", error=str(e))
+        return "down"
+
 
-    # Check Redis
+async def _check_redis() -> str:
     redis_url = os.getenv("REDIS_URL", "redis://localhost:6379")
+    client = redis.from_url(redis_url)
+    try:
+        await asyncio.wait_for(client.ping(), timeout=2.0)
+        return "up"
+    except Exception as e:
+        logger.error("readyz_redis_failed", error=str(e))
+        return "down"
+    finally:
+        await client.close()
+
+
+async def _check_soroban_rpc() -> str:
+    rpc_url = os.getenv("STELLAR_SOROBAN_RPC_URL")
+    if not rpc_url:
+        logger.error("readyz_soroban_rpc_missing")
+        return "down"
+
+    payload = {"jsonrpc": "2.0", "id": 1, "method": "getHealth"}
+    timeout = aiohttp.ClientTimeout(total=2.0)
     try:
-        r = redis.from_url(redis_url)
-        await asyncio.wait_for(r.ping(), timeout=2.0)
-        await r.close()
+        async with aiohttp.ClientSession(timeout=timeout) as session:
+            async with session.post(rpc_url, json=payload) as response:
+                if response.status >= 400:
+                    logger.error(
+                        "readyz_soroban_rpc_http_failed",
+                        status_code=response.status,
+                    )
+                    return "down"
+                body = await response.json()
+                if body.get("error"):
+                    logger.error("readyz_soroban_rpc_error", error=body["error"])
+                    return "down"
+                return "up"
     except Exception as e:
-        logger.error("health_check_redis_failed", error=str(e))
-        health_status["redis"] = "down"
-        is_healthy = False
+        logger.error("readyz_soroban_rpc_failed", error=str(e))
+        return "down"
+
+
+@app.get("/livez", tags=["Health"], summary="Liveness check endpoint")
+async def livez() -> dict[str, str]:
+    """Return 200 when the process can accept HTTP requests."""
+    return {"status": "alive"}
+
+
+@app.get("/health", tags=["Health"], summary="Compatibility health check endpoint")
+async def health_check() -> dict[str, str]:
+    """Backward-compatible liveness endpoint for existing curl checks."""
+    return await livez()
+
+
+@app.get("/readyz", tags=["Health"], summary="Readiness check endpoint")
+async def readyz(response: Response, db: Session = Depends(get_database)) -> dict[str, str]:
+    """Return 200 only when database, Redis, and Soroban RPC probes pass."""
+    health_status = {
+        "status": "ready",
+        "database": await _check_database(db),
+        "redis": await _check_redis(),
+        "soroban_rpc": await _check_soroban_rpc(),
+    }
 
-    if not is_healthy:
+    if any(value != "up" for key, value in health_status.items() if key != "status"):
         health_status["status"] = "degraded"
         response.status_code = 503
 

quantara/web_app/tests/test_health_routes_static.py

@@ -0,0 +1,32 @@
+"""Static coverage for split liveness/readiness routes."""
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+MAIN_SOURCE = (ROOT / "web_app" / "api" / "main.py").read_text(encoding="utf-8")
+COMPOSE_SOURCE = (ROOT.parent / "devops" / "docker-compose.quantara.yaml").read_text(
+    encoding="utf-8"
+)
+
+
+def test_liveness_and_compatibility_routes_are_separate_from_readiness():
+    assert '@app.get("/livez"' in MAIN_SOURCE
+    assert '@app.get("/health"' in MAIN_SOURCE
+    assert 'return await livez()' in MAIN_SOURCE
+    assert '@app.get("/readyz"' in MAIN_SOURCE
+    assert 'response.status_code = 503' in MAIN_SOURCE
+
+
+def test_readiness_checks_database_redis_and_soroban_rpc():
+    assert "async def _check_database" in MAIN_SOURCE
+    assert "async def _check_redis" in MAIN_SOURCE
+    assert "async def _check_soroban_rpc" in MAIN_SOURCE
+    assert 'text("SELECT 1")' in MAIN_SOURCE
+    assert "client.ping()" in MAIN_SOURCE
+    assert 'os.getenv("STELLAR_SOROBAN_RPC_URL")' in MAIN_SOURCE
+    assert '"method": "getHealth"' in MAIN_SOURCE
+
+
+def test_backend_healthcheck_uses_readiness_endpoint():
+    assert "http://localhost:8000/readyz" in COMPOSE_SOURCE
+    assert "http://localhost:8000/health" not in COMPOSE_SOURCE