Skip to content

Commit ffb59cd

Browse files
Par-tPar-t
committed
Add locust-based Kubernetes load tests and HPA benchmarks
1 parent a37b201 commit ffb59cd

4 files changed

Lines changed: 221 additions & 1 deletion

File tree

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,4 @@ outputs/
3535

3636
# Plan (personal notes)
3737
plan.md
38-
notes.md
38+
notes.mdk8s/results/

‎k8s/load_test.sh‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#!/bin/bash
2+
# Kubernetes load test for the recommendation API.
3+
#
4+
# Runs two scenarios using Locust and saves CSV results:
5+
# Scenario 1 — Baseline: 10 users, 60s (measure steady-state latency)
6+
# Scenario 2 — Stress: 50 users, 120s (trigger HPA, measure degradation)
7+
#
8+
# Prerequisites:
9+
# - Minikube running: minikube start --driver=docker
10+
# - Mounts active: minikube mount $(pwd)/outputs:/mnt/outputs
11+
# minikube mount $(pwd)/data/processed:/mnt/data/processed
12+
# - App deployed: kubectl apply -f k8s/deployment.yaml -f k8s/service.yaml
13+
# - HPA deployed: kubectl apply -f k8s/hpa.yaml
14+
# - metrics-server: minikube addons enable metrics-server
15+
# - locust installed: pip install locust
16+
#
17+
# Usage (from project root):
18+
# bash k8s/load_test.sh
19+
20+
set -e
21+
22+
RESULTS_DIR="k8s/results"
23+
mkdir -p "$RESULTS_DIR"
24+
25+
# Get Minikube service URL
26+
# On macOS with Docker driver, `minikube service --url` blocks (it's a tunnel).
27+
# Accept the URL as an argument or prompt the user to provide it.
28+
if [ -n "$1" ]; then
29+
SERVICE_URL="$1"
30+
else
31+
echo "Usage: bash k8s/load_test.sh <SERVICE_URL>"
32+
echo ""
33+
echo "To get the URL, run this in a separate terminal and leave it open:"
34+
echo " minikube service recommendation-api --url"
35+
echo ""
36+
echo "Then pass the printed URL here, e.g.:"
37+
echo " bash k8s/load_test.sh http://127.0.0.1:12345"
38+
exit 1
39+
fi
40+
echo "Service URL: $SERVICE_URL"
41+
echo ""
42+
43+
# Verify service is healthy before running load tests
44+
echo "Verifying service health..."
45+
HEALTH=$(curl -s --max-time 5 "$SERVICE_URL/health" 2>/dev/null || echo "")
46+
if ! echo "$HEALTH" | grep -q '"model_loaded":true'; then
47+
echo "ERROR: Service is not healthy. Response: $HEALTH"
48+
exit 1
49+
fi
50+
echo "Health check passed: $HEALTH"
51+
echo ""
52+
53+
# ─────────────────────────────────────────────────────────────
54+
# Scenario 1 — Baseline (10 users, 60s)
55+
# ─────────────────────────────────────────────────────────────
56+
echo "========================================"
57+
echo "Scenario 1: Baseline (10 users, 60s)"
58+
echo "========================================"
59+
60+
echo "Pod count before:"
61+
kubectl get pods -l app=recommendation-api --no-headers | wc -l | xargs echo " Pods:"
62+
63+
locust -f k8s/locustfile.py \
64+
--headless \
65+
--users 10 \
66+
--spawn-rate 2 \
67+
--run-time 60s \
68+
--host "$SERVICE_URL" \
69+
--csv "$RESULTS_DIR/baseline" \
70+
--only-summary \
71+
2>&1 | tail -20
72+
73+
echo ""
74+
echo "Baseline results saved to $RESULTS_DIR/baseline_stats.csv"
75+
echo ""
76+
77+
# Parse and print key metrics
78+
python3 - "$RESULTS_DIR/baseline_stats.csv" << 'PYEOF'
79+
import sys, csv
80+
with open(sys.argv[1]) as f:
81+
rows = list(csv.DictReader(f))
82+
for row in rows:
83+
if row.get("Name") == "/recommend":
84+
p50 = row.get("50%", "N/A")
85+
p95 = row.get("95%", "N/A")
86+
rps = row.get("Requests/s", "N/A")
87+
fail = row.get("Failure Count", "0")
88+
print(f" /recommend p50={p50}ms p95={p95}ms RPS={rps} failures={fail}")
89+
PYEOF
90+
91+
# ─────────────────────────────────────────────────────────────
92+
# Scenario 2 — Stress (50 users, 120s)
93+
# ─────────────────────────────────────────────────────────────
94+
echo ""
95+
echo "========================================"
96+
echo "Scenario 2: Stress (50 users, 120s)"
97+
echo "========================================"
98+
99+
echo "Pod count before stress test:"
100+
kubectl get pods -l app=recommendation-api --no-headers | wc -l | xargs echo " Pods:"
101+
102+
echo "HPA state before stress test:"
103+
kubectl get hpa recommendation-api --no-headers 2>/dev/null || echo " HPA not deployed"
104+
echo ""
105+
106+
locust -f k8s/locustfile.py \
107+
--headless \
108+
--users 50 \
109+
--spawn-rate 5 \
110+
--run-time 120s \
111+
--host "$SERVICE_URL" \
112+
--csv "$RESULTS_DIR/stress" \
113+
--only-summary \
114+
2>&1 | tail -20
115+
116+
echo ""
117+
echo "Stress results saved to $RESULTS_DIR/stress_stats.csv"
118+
echo ""
119+
120+
# Parse and print key metrics
121+
python3 - "$RESULTS_DIR/stress_stats.csv" << 'PYEOF'
122+
import sys, csv
123+
with open(sys.argv[1]) as f:
124+
rows = list(csv.DictReader(f))
125+
for row in rows:
126+
if row.get("Name") == "/recommend":
127+
p50 = row.get("50%", "N/A")
128+
p95 = row.get("95%", "N/A")
129+
rps = row.get("Requests/s", "N/A")
130+
fail = row.get("Failure Count", "0")
131+
print(f" /recommend p50={p50}ms p95={p95}ms RPS={rps} failures={fail}")
132+
PYEOF
133+
134+
echo "Pod count after stress test (HPA may have scaled up):"
135+
kubectl get pods -l app=recommendation-api
136+
echo ""
137+
echo "HPA state after stress test:"
138+
kubectl get hpa recommendation-api 2>/dev/null || echo " HPA not deployed"
139+
140+
echo ""
141+
echo "========================================"
142+
echo "Load test complete."
143+
echo "Results: $RESULTS_DIR/baseline_stats.csv $RESULTS_DIR/stress_stats.csv"
144+
echo "========================================"

‎k8s/locustfile.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
"""
2+
Locust load test for the recommendation API.
3+
4+
Tasks:
5+
- GET /health (lightweight, ~10% of requests)
6+
- POST /recommend (main workload, ~90% of requests)
7+
8+
The recommend task samples a random valid user_id from the local
9+
data/processed/user_to_idx.json file, so every request is realistic.
10+
11+
Usage (run from project root):
12+
locust -f k8s/locustfile.py --host http://<MINIKUBE_URL>
13+
14+
Or headless (used by load_test.sh):
15+
locust -f k8s/locustfile.py --headless -u 10 -r 2 --run-time 60s \
16+
--host http://<MINIKUBE_URL> --csv k8s/results/baseline
17+
"""
18+
19+
import json
20+
import random
21+
from pathlib import Path
22+
23+
from locust import HttpUser, between, task
24+
25+
# Load valid user IDs once at module level so all workers share the same list
26+
_USER_IDS_PATH = Path("data/processed/user_to_idx.json")
27+
if _USER_IDS_PATH.exists():
28+
with open(_USER_IDS_PATH) as f:
29+
_USER_IDS = list(json.load(f).keys())
30+
else:
31+
# Fallback: empty list — recommend tasks will be skipped gracefully
32+
_USER_IDS = []
33+
34+
35+
class RecommendationUser(HttpUser):
36+
"""
37+
Simulates a client calling the recommendation API.
38+
39+
wait_time: pause 100ms–500ms between requests per user
40+
(realistic for a frontend polling for recommendations)
41+
"""
42+
43+
wait_time = between(0.1, 0.5)
44+
45+
@task(1)
46+
def health_check(self):
47+
"""Lightweight health poll — low weight, keeps the ratio realistic."""
48+
self.client.get("/health", name="/health")
49+
50+
@task(9)
51+
def get_recommendations(self):
52+
"""POST /recommend with a random valid user."""
53+
if not _USER_IDS:
54+
return
55+
56+
user_id = random.choice(_USER_IDS)
57+
payload = {"user_id": user_id, "top_k": 10}
58+
59+
with self.client.post(
60+
"/recommend",
61+
json=payload,
62+
name="/recommend",
63+
catch_response=True,
64+
) as response:
65+
if response.status_code == 200:
66+
response.success()
67+
elif response.status_code == 404:
68+
# User not found in this model version — not a failure
69+
response.success()
70+
else:
71+
response.failure(
72+
f"Unexpected status {response.status_code}: {response.text[:100]}"
73+
)

‎requirements.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ uvicorn>=0.27.0
1919
# Metrics & baselines
2020
scipy>=1.11.0
2121
scikit-learn>=1.4.0
22+
23+
# Load testing
24+
locust>=2.20.0

0 commit comments

Comments
 (0)