diff --git a/README.md b/README.md index dc0ac749..802e4142 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,13 @@ pnpm agentx dlq list # List failed tasks pnpm agentx dlq size # Show DLQ size pnpm agentx dlq clear # Clear DLQ -# Graceful shutdown (NEW!) +# Graceful shutdown pnpm agentx shutdown "Maintenance" # Trigger graceful shutdown +# Load Testing +pnpm test:load # Run performance load tests (100-1000 users) +pnpm test:load:shutdown # Test graceful shutdown under load + # Other commands pnpm agentx config # Manage configuration pnpm agentx cost # Show cost analysis diff --git a/load-tests/README.md b/load-tests/README.md new file mode 100644 index 00000000..53e9edae --- /dev/null +++ b/load-tests/README.md @@ -0,0 +1,238 @@ +# AgentX Load Tests + +Performance and scalability testing suite using [k6](https://k6.io/). + +## Prerequisites + +### Install k6 + +**macOS:** + +```bash +brew install k6 +``` + +**Linux:** + +```bash +curl https://k6.io/install.sh | sudo bash +``` + +**Windows:** + +```bash +choco install k6 +``` + +**Docker:** + +```bash +docker run --rm grafana/k6 version +``` + +## Running Load Tests + +### Performance Test (100-1000 concurrent users) + +```bash +# Basic run +k6 run load-tests/performance.test.js + +# With custom base URL +BASE_URL=http://localhost:3000 k6 run load-tests/performance.test.js + +# With API key +BASE_URL=http://localhost:3000 API_KEY=your-key k6 run load-tests/performance.test.js + +# Generate HTML report +k6 run load-tests/performance.test.js | tee summary.json +k6 convert summary.json --output report.html +``` + +### Graceful Shutdown Test + +```bash +# Test shutdown under load +k6 run load-tests/shutdown.test.js + +# With custom URL +BASE_URL=http://localhost:3000 k6 run load-tests/shutdown.test.js +``` + +## Test Scenarios + +### Performance Test + +**Objective:** Validate system performance under increasing load + +**Stages:** + +1. **Ramp-up** (30s): 0 → 100 users +2. **Steady** (1m): 100 users +3. **Ramp-up** (30s): 100 → 500 users +4. **Steady** (2m): 500 users +5. **Ramp-up** (30s): 500 → 1000 users (peak) +6. **Peak** (3m): 1000 users +7. **Ramp-down** (30s): 1000 → 0 users + +**User Behavior:** + +- 70% submit tasks +- 20% check task status +- 10% list all tasks + +### Graceful Shutdown Test + +**Objective:** Verify system handles shutdown gracefully under load + +**Stages:** + +1. **Ramp-up** (30s): 0 → 50 users +2. **Steady** (1m): 50 users +3. **Shutdown** (10s): Trigger SIGTERM, verify in-flight completion + +## Performance Budgets + +| Metric | Threshold | Description | +| --------------- | -------------- | --------------------------------- | +| p95 Latency | < 500ms | 95% of requests faster than 500ms | +| p99 Latency | < 1000ms | 99% of requests faster than 1s | +| Error Rate | < 1% | Less than 1% failed requests | +| Task Submission | < 500ms (p95) | Task creation latency | +| Task Completion | < 1000ms (p99) | Task status check latency | + +## Interpreting Results + +### k6 Output Example + +``` + ✓ http_req_duration + ✓ p(95)<500 + ✓ p(99)<1000 + ✓ errors + ✓ rate<0.01 +``` + +✅ **All thresholds met** - System is performing within budget +❌ **Threshold failed** - System needs optimization + +### Key Metrics + +**HTTP Request Duration:** + +- `avg`: Average response time +- `min`: Fastest response +- `med`: Median response time +- `p(90)`: 90th percentile +- `p(95)`: 95th percentile (performance budget) +- `p(99)`: 99th percentile (performance budget) +- `max`: Slowest response + +**Error Rate:** + +- Percentage of failed requests +- Should be < 1% for production readiness + +**Throughput:** + +- Requests per second (reqs/s) +- Higher is better, but watch for saturation + +## CI/CD Integration + +### GitHub Actions + +```yaml +- name: Run Load Tests + run: | + npm install -g k6 + k6 run load-tests/performance.test.js + +- name: Upload Load Test Results + uses: actions/upload-artifact@v3 + with: + name: load-test-results + path: load-tests/results/ +``` + +### Docker + +```yaml +services: + k6: + image: grafana/k6:latest + volumes: + - ./load-tests:/scripts + environment: + - BASE_URL=http://api:3000 + command: run /scripts/performance.test.js +``` + +## Troubleshooting + +### High Latency + +**Symptoms:** p95 or p99 exceeds budget + +**Actions:** + +1. Check database query performance +2. Review LLM provider response times +3. Analyze resource utilization (CPU, memory) +4. Check for connection pool exhaustion + +### High Error Rate + +**Symptoms:** Error rate > 1% + +**Actions:** + +1. Check application logs for errors +2. Verify database connections +3. Review rate limiting configuration +4. Check LLM provider availability + +### Test Fails to Connect + +**Symptoms:** Connection refused errors + +**Actions:** + +1. Verify target system is running: `curl http://localhost:3000/health` +2. Check firewall rules +3. Verify BASE_URL environment variable +4. Ensure system has capacity for load test + +## Performance Optimization Tips + +1. **Database:** + - Add indexes for frequently queried fields + - Use connection pooling + - Implement query caching + +2. **LLM Providers:** + - Implement request batching + - Use response caching + - Configure appropriate timeouts + +3. **Application:** + - Enable compression + - Use CDN for static assets + - Implement rate limiting + +4. **Infrastructure:** + - Scale horizontally under load + - Use auto-scaling groups + - Implement circuit breakers + +## References + +- [k6 Documentation](https://k6.io/docs/) +- [Performance Testing Best Practices](https://k6.io/docs/guides/) +- [Thresholds & Assertions](https://k6.io/docs/using-k6/thresholds/) + +--- + +**Last Updated:** July 26, 2026 +**Test Suite Version:** 1.0.0 +**k6 Version:** 0.45+ diff --git a/load-tests/performance.test.js b/load-tests/performance.test.js new file mode 100644 index 00000000..7ce1ae29 --- /dev/null +++ b/load-tests/performance.test.js @@ -0,0 +1,247 @@ +/** + * AgentX Load Test Suite + * Performance and scalability testing with k6 + * + * Install k6: https://k6.io/docs/getting-started/installation/ + * Run: k6 run load-tests/performance.test.js + */ + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { Rate, Trend } from 'k6/metrics'; + +// Custom metrics +const errorRate = new Rate('errors'); +const taskSubmissionLatency = new Trend('task_submission_latency'); +const taskCompletionLatency = new Trend('task_completion_latency'); + +// Performance budgets +const PERF_BUDGETS = { + p95Latency: 500, // ms + p99Latency: 1000, // ms + errorRate: 0.01, // 1% +}; + +// Test configuration +export const options = { + // Stage 1: Ramp up to 100 users + stages: [ + { duration: '30s', target: 100 }, // Ramp to 100 users + { duration: '1m', target: 100 }, // Stay at 100 users + { duration: '30s', target: 500 }, // Ramp to 500 users + { duration: '2m', target: 500 }, // Stay at 500 users + { duration: '30s', target: 1000 }, // Ramp to 1000 users + { duration: '3m', target: 1000 }, // Stay at 1000 users (peak load) + { duration: '30s', target: 0 }, // Ramp down to 0 + ], + + thresholds: { + 'http_req_duration': [`p(95)<${PERF_BUDGETS.p95Latency}`, `p(99)<${PERF_BUDGETS.p99Latency}`], + 'errors': [`rate<${PERF_BUDGETS.errorRate}`], + 'task_submission_latency': [`p(95)<${PERF_BUDGETS.p95Latency}`], + 'task_completion_latency': [`p(99)<${PERF_BUDGETS.p99Latency}`], + }, + + // Scenarios for different user behaviors + scenarios: { + // 70% users submit tasks + submit_task: { + executor: 'ramping-vus', + exec: 'submitTask', + startVUs: 0, + stages: [ + { duration: '30s', target: 70 }, + { duration: '2m', target: 70 }, + { duration: '30s', target: 0 }, + ], + }, + + // 20% users check status + check_status: { + executor: 'ramping-vus', + exec: 'checkStatus', + startVUs: 0, + stages: [ + { duration: '30s', target: 20 }, + { duration: '2m', target: 20 }, + { duration: '30s', target: 0 }, + ], + }, + + // 10% users list all tasks + list_tasks: { + executor: 'ramping-vus', + exec: 'listTasks', + startVUs: 0, + stages: [ + { duration: '30s', target: 10 }, + { duration: '2m', target: 10 }, + { duration: '30s', target: 0 }, + ], + }, + }, +}; + +// Test data +const BASE_URL = __ENV.BASE_URL || 'http://localhost:3000'; +const API_KEY = __ENV.API_KEY || 'test-api-key'; + +const commonHeaders = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${API_KEY}`, +}; + +/** + * Scenario 1: Submit a new task + * Simulates user submitting a goal for agent execution + */ +export function submitTask() { + const startTime = Date.now(); + + const payloads = [ + { goal: 'Write a hello world function', role: 'coder' }, + { goal: 'Review this code for security issues', role: 'reviewer' }, + { goal: 'Write unit tests for user service', role: 'tester' }, + { goal: 'Check for SQL injection vulnerabilities', role: 'security' }, + { goal: 'Refactor the authentication module', role: 'coder' }, + ]; + + const payload = payloads[Math.floor(Math.random() * payloads.length)]; + + const params = { + headers: commonHeaders, + tags: { name: 'SubmitTask' }, + }; + + const res = http.post(`${BASE_URL}/api/v1/tasks/submit`, JSON.stringify(payload), params); + + const latency = Date.now() - startTime; + taskSubmissionLatency.add(latency); + + const success = check(res, { + 'task submission status is 200 or 201': (r) => r.status === 200 || r.status === 201, + 'task submission returns task ID': (r) => { + try { + const body = JSON.parse(r.body); + return body.taskId !== undefined; + } catch { + return false; + } + }, + }); + + errorRate.add(!success); + + sleep(1); // Think time +} + +/** + * Scenario 2: Check task status + * Simulates user polling for task completion + */ +export function checkStatus() { + const startTime = Date.now(); + + // Use a mix of task IDs (some real, some fake to test error handling) + const taskIds = [ + 'test-task-1', + 'test-task-2', + 'test-task-3', + 'nonexistent-task', + ]; + + const taskId = taskIds[Math.floor(Math.random() * taskIds.length)]; + + const params = { + headers: commonHeaders, + tags: { name: 'CheckStatus' }, + }; + + const res = http.get(`${BASE_URL}/api/v1/tasks/${taskId}`, params); + + const latency = Date.now() - startTime; + taskCompletionLatency.add(latency); + + const success = check(res, { + 'status check returns 200 or 404': (r) => r.status === 200 || r.status === 404, + 'status check response is valid JSON': (r) => { + try { + JSON.parse(r.body); + return true; + } catch { + return false; + } + }, + }); + + errorRate.add(!success); + + sleep(0.5); // Think time +} + +/** + * Scenario 3: List all tasks + * Simulates user viewing task dashboard + */ +export function listTasks() { + const startTime = Date.now(); + + const params = { + headers: commonHeaders, + tags: { name: 'ListTasks' }, + }; + + const res = http.get(`${BASE_URL}/api/v1/tasks`, params); + + const latency = Date.now() - startTime; + taskCompletionLatency.add(latency); + + const success = check(res, { + 'list tasks status is 200': (r) => r.status === 200, + 'list tasks returns array': (r) => { + try { + const body = JSON.parse(r.body); + return Array.isArray(body.tasks); + } catch { + return false; + } + }, + }); + + errorRate.add(!success); + + sleep(2); // Think time +} + +/** + * Setup: Create test data before load test + */ +export function setup() { + console.log('Setting up load test...'); + console.log(`Base URL: ${BASE_URL}`); + console.log(`Performance Budgets:`); + console.log(` - p95 Latency: < ${PERF_BUDGETS.p95Latency}ms`); + console.log(` - p99 Latency: < ${PERF_BUDGETS.p99Latency}ms`); + console.log(` - Error Rate: < ${(PERF_BUDGETS.errorRate * 100).toFixed(1)}%`); + + // Health check + const healthRes = http.get(`${BASE_URL}/health`); + if (healthRes.status !== 200) { + throw new Error(`Target system is not healthy: ${healthRes.status}`); + } + + console.log('Target system is healthy ✓'); + + return { startTime: Date.now() }; +} + +/** + * Teardown: Cleanup after load test + */ +export function teardown(data) { + const duration = Date.now() - data.startTime; + console.log(`\nLoad test completed in ${(duration / 1000).toFixed(1)}s`); + console.log('\nPerformance Summary:'); + console.log(' Check k6 HTML report for detailed metrics'); + console.log(' Key metrics: p95 latency, p99 latency, error rate'); +} \ No newline at end of file diff --git a/load-tests/shutdown.test.js b/load-tests/shutdown.test.js new file mode 100644 index 00000000..7e658dfc --- /dev/null +++ b/load-tests/shutdown.test.js @@ -0,0 +1,107 @@ +/** + * Graceful Shutdown Load Test + * Tests system behavior during shutdown under load + */ + +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import { Rate } from 'k6/metrics'; + +const errorRate = new Rate('shutdown_errors'); + +export const options = { + stages: [ + { duration: '30s', target: 50 }, // Ramp to 50 users + { duration: '1m', target: 50 }, // Steady state + { duration: '10s', target: 50 }, // Maintain during shutdown signal + ], + + thresholds: { + 'errors': ['rate<0.05'], // 5% error rate allowed during shutdown + }, +}; + +const BASE_URL = __ENV.BASE_URL || 'http://localhost:3000'; +const API_KEY = __ENV.API_KEY || 'test-api-key'; + +const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${API_KEY}`, +}; + +/** + * Test graceful shutdown under load + * + * This test: + * 1. Starts with steady load (50 VUs submitting tasks) + * 2. Triggers graceful shutdown mid-test + * 3. Verifies in-flight tasks complete + * 4. Verifies no data corruption + */ +export default function () { + const startTime = Date.now(); + + // Submit task + const payload = { + goal: `Load test task ${__VU}-${__ITER}`, + role: 'coder', + }; + + const res = http.post(`${BASE_URL}/api/v1/tasks/submit`, JSON.stringify(payload), { + headers, + tags: { name: 'SubmitDuringShutdown' }, + }); + + const success = check(res, { + 'task submitted or graceful rejection': (r) => { + // Accept 200 (success), 503 (shutdown in progress), or 429 (rate limit) + return [200, 503, 429].includes(r.status); + }, + }); + + errorRate.add(!success); + + // Check task status if submitted successfully + if (res.status === 200) { + try { + const body = JSON.parse(res.body); + if (body.taskId) { + sleep(0.1); + + const statusRes = http.get(`${BASE_URL}/api/v1/tasks/${body.taskId}`, { + headers, + tags: { name: 'CheckDuringShutdown' }, + }); + + check(statusRes, { + 'task status retrievable': (r) => r.status === 200 || r.status === 404, + }); + } + } catch (e) { + // Ignore parse errors + } + } + + sleep(0.5); +} + +export function handleSummary(data) { + return { + 'stdout': textSummary(data, { indent: ' ', enableColors: true }), + 'load-tests/results/shutdown-test.json': JSON.stringify(data), + }; +} + +function textSummary(data, options) { + const { metrics } = data; + return ` +Graceful Shutdown Test Results: +================================ +Iterations: ${data.metrics.iterations.values.count} +HTTP Requests: ${data.metrics.http_reqs.values.count} +Error Rate: ${(metrics.shutdown_errors.values.rate * 100).toFixed(2)}% +Avg Response Time: ${metrics.http_req_duration.values.avg.toFixed(0)}ms +p95 Response Time: ${metrics.http_req_duration.values['p(95)'].toFixed(0)}ms +p99 Response Time: ${metrics.http_req_duration.values['p(99)'].toFixed(0)}ms +`; +} \ No newline at end of file diff --git a/package.json b/package.json index 2cb93151..cc70e487 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,8 @@ "test:watch": "turbo run test -- --watch", "test:coverage": "turbo run test:coverage", "test:e2e": "vitest run --config tests/e2e/vitest.e2e.config.ts", + "test:load": "k6 run load-tests/performance.test.js", + "test:load:shutdown": "k6 run load-tests/shutdown.test.js", "lint": "turbo run lint", "format": "prettier --write \"**/*.{ts,tsx,md,json,yaml,yml}\"", "format:check": "prettier --check \"**/*.{ts,tsx,md,json,yaml,yml}\"",