Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions backend/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,11 @@ SOROBAN_RPC_URL=https://soroban-testnet.stellar.org:443
# Additional Configuration
# Add other configuration variables as needed
ENVIRONMENT=development

# Monitoring Configuration
# Enable/disable metrics collection (true/false)
METRICS_ENABLED=true
# Prometheus metrics endpoint path
METRICS_PATH=/metrics
# HTTP metrics endpoint path (auto-enabled by actix-web-prom)
HTTP_METRICS_PATH=/metrics/http
Comment thread
Godfrey-Delight marked this conversation as resolved.
88 changes: 88 additions & 0 deletions backend/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,94 @@ For questions or issues:

---

## 📊 Monitoring & Observability

XLMate backend includes comprehensive Prometheus metrics and Grafana dashboards for monitoring system health and performance.

### Metrics Endpoints

The backend exposes metrics at two endpoints:

- **`/metrics`** - Custom application metrics (active games, WebSocket connections, etc.)
- **`/metrics/http`** - HTTP request metrics (request rate, duration, status codes)

### Available Metrics

#### Application Metrics
- `xlmate_active_games` - Current number of active games (Gauge)
- `xlmate_ws_connections` - Active WebSocket connections (Gauge)
- `xlmate_db_query_duration_seconds` - Database query latency (Histogram)
- `xlmate_matchmaking_queue_size` - Players waiting in matchmaking queue (Gauge)
- `xlmate_ai_requests_total` - Total AI analysis requests (Counter)
- `xlmate_auth_events_total` - Authentication events (Counter)
- `xlmate_game_events_total` - Game lifecycle events (Counter)

#### HTTP Metrics (via actix-web-prom)
- `xlmate_http_requests_total` - HTTP request count by method and status
- `xlmate_http_request_duration_seconds` - HTTP request duration (Histogram)

### Running with Monitoring

Start the full monitoring stack with Docker Compose:

```bash
docker-compose up -d
```

This will start:
- **PostgreSQL** on port 5432
- **Redis** on port 6379
- **Prometheus** on port 9090
- **Grafana** on port 3000

### Accessing Dashboards

- **Prometheus**: http://localhost:9090
- **Grafana**: http://localhost:3000
- Username: `admin`
- Password: `admin`
- **Metrics Endpoint**: http://localhost:8080/metrics

### Grafana Dashboards

The Grafana instance comes pre-configured with:
- Prometheus datasource
- XLMate Backend Monitoring dashboard with panels for:
- Active Games (stat panel)
- WebSocket Connections (stat panel)
- Database Query Latency P50/P95 (time series)
- HTTP Request Rate (time series)
- Game Events Over Time (time series)
- AI Request Rate (time series)

### Custom Queries

You can query metrics directly in Prometheus or Grafana:

```promql
# Current active games
xlmate_active_games

# WebSocket connection rate
rate(xlmate_ws_connections[5m])

# 95th percentile database query latency
histogram_quantile(0.95, rate(xlmate_db_query_duration_seconds_bucket[5m]))

# HTTP error rate
rate(xlmate_http_requests_total{status=~"5.."}[5m])
```

### Testing Metrics

Run the metrics unit tests:

```bash
cargo test metrics_tests
```

---

## 📋 Quick Reference

| Document | Purpose | Read Time |
Expand Down
5 changes: 5 additions & 0 deletions backend/modules/api/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ st_core = { path = "../st_core", features = ["api"] }
# For Redis Pub/Sub in WebSocket
redis = { version = "0.24", features = ["tokio-comp", "json"] }

# For Prometheus metrics
prometheus = "0.13"
actix-web-prom = "0.7"
once_cell = "1.19"

[dev-dependencies]
tokio = { version = "1", features = ["full"] }
actix-rt = "2.9"
Expand Down
7 changes: 7 additions & 0 deletions backend/modules/api/src/ai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use validator::Validate;

use service::engine_service::EngineService;
use std::env;
use crate::metrics::increment_ai_requests;

#[utoipa::path(
post,
Expand All @@ -28,6 +29,9 @@ use std::env;
)]
#[post("/suggest")]
pub async fn get_ai_suggestion(payload: Json<AiSuggestionRequest>) -> HttpResponse {
// Track AI request
increment_ai_requests("suggestion");

match payload.0.validate() {
Ok(_) => {
let engine_path = env::var("ENGINE_PATH").unwrap_or_else(|_| "stockfish".to_string());
Expand Down Expand Up @@ -90,6 +94,9 @@ pub async fn get_ai_suggestion(payload: Json<AiSuggestionRequest>) -> HttpRespon
)]
#[post("/analyze")]
pub async fn analyze_position(payload: Json<PositionAnalysisRequest>) -> HttpResponse {
// Track AI request
increment_ai_requests("analysis");

match payload.0.validate() {
Ok(_) => {
let engine_path = env::var("ENGINE_PATH").unwrap_or_else(|_| "stockfish".to_string());
Expand Down
8 changes: 8 additions & 0 deletions backend/modules/api/src/auth.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use uuid::Uuid;
use dto::auth::{RegisterRequest, LoginRequest, AuthResponse, ErrorResponse, RefreshTokenRequest, RefreshResponse, LogoutResponse};
use security::{JwtService, TokenService, TokenServiceError};
use sea_orm::DatabaseConnection;
use crate::metrics::increment_auth_events;

/// Register a new user
#[utoipa::path(
Expand All @@ -32,6 +33,8 @@ pub async fn register(
}

// For now, return a mock response
increment_auth_events("register", true);

HttpResponse::Created().json(AuthResponse {
access_token: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...".to_string(),
refresh_token: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...".to_string(),
Expand Down Expand Up @@ -77,6 +80,7 @@ pub async fn login(
let access_token = match jwt_service.generate_token(user_id, &username) {
Ok(t) => t,
Err(_) => {
increment_auth_events("login", false);
return HttpResponse::InternalServerError().json(ErrorResponse {
message: "Failed to generate access token".to_string(),
code: "TOKEN_ERROR".to_string(),
Expand Down Expand Up @@ -123,6 +127,10 @@ pub async fn login(
.finish();

response.add_cookie(&cookie).ok();

// Track successful login
increment_auth_events("login", true);

response
}

Expand Down
33 changes: 25 additions & 8 deletions backend/modules/api/src/games.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use validator::Validate;
use uuid::Uuid;
use sea_orm::DatabaseConnection;
use service::games::GameService;
use crate::metrics::{increment_active_games, decrement_active_games, increment_game_events};

// ---------------------------------------------------------------------------
// Helper: extract authenticated player UUID inserted by the JWT middleware.
Expand Down Expand Up @@ -62,10 +63,16 @@ pub async fn create_game(
};

match GameService::create_game(db.get_ref(), creator_id, payload.0).await {
Ok(game_dto) => HttpResponse::Created().json(json!({
"message": "Game created successfully",
"data": { "game": game_dto }
})),
Ok(game_dto) => {
// Track game creation metric
increment_active_games();
increment_game_events("created");

HttpResponse::Created().json(json!({
"message": "Game created successfully",
"data": { "game": game_dto }
}))
}
Err(e) => {
eprintln!("create_game error: {e}");
HttpResponse::InternalServerError().json(json!({
Expand Down Expand Up @@ -340,10 +347,16 @@ pub async fn abandon_game(
let game_id = id.into_inner();

match GameService::abandon_game(db.get_ref(), game_id, player_id).await {
Ok(_) => HttpResponse::Ok().json(json!({
"message": "Game abandoned successfully",
"data": {}
})),
Ok(_) => {
// Track game abandonment metric
decrement_active_games();
increment_game_events("abandoned");

HttpResponse::Ok().json(json!({
"message": "Game abandoned successfully",
"data": {}
}))
}
Err(ApiError::NotFound(_)) => HttpResponse::NotFound().json(json!({
"message": "Game not found"
})),
Expand Down Expand Up @@ -540,6 +553,10 @@ pub async fn complete_game(
// Complete the game and update ratings
match GameService::complete_game(db.get_ref(), game_id, result_enum.clone(), Some(rating_config)).await {
Ok((white_new_rating, black_new_rating)) => {
// Track game completion metric
decrement_active_games();
increment_game_events("completed");

let white_change = white_new_rating - white_old_rating;
let black_change = black_new_rating - black_old_rating;

Expand Down
4 changes: 4 additions & 0 deletions backend/modules/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ pub mod config;
pub mod server;
pub mod players;
pub mod games;
pub mod metrics;

#[cfg(test)]
mod metrics_tests;

// External modules
extern crate challenge;
Expand Down
Loading