diff --git a/.dockerignore b/.dockerignore index fd679bd..c957f33 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ !ui !ui/** !server.py +!metrics.py !Dockerfile **/__pycache__/ **/*.py[cod] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caa2fd7..675fb20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" - - name: Syntax-check UI server - run: python3 -m py_compile server.py scripts/patch-idle-cpu.py + - name: Syntax-check Python entrypoints + run: python3 -m py_compile server.py metrics.py scripts/patch-idle-cpu.py python-tests: runs-on: ubuntu-latest @@ -46,6 +46,8 @@ jobs: run: python3 tests/test_auto_backup.py - name: Server-control (systemctl dispatch) tests run: python3 tests/test_server_control.py + - name: Metrics exporter tests + run: python3 tests/test_metrics.py - name: ServerDescription schema tests run: python3 tests/test_schema.py - name: HTTP integration tests @@ -102,7 +104,7 @@ jobs: - uses: actions/checkout@v4 - name: Validate example JSON run: | - python3 -c "import json,sys; [json.load(open(f)) for f in ['scripts/ServerDescription_example.json','scripts/WorldDescription_example.json']]" + python3 -c "import json,sys; [json.load(open(f)) for f in ['scripts/ServerDescription_example.json','scripts/WorldDescription_example.json','helm/windrose/dashboards/windrose-overview.json']]" kustomize-render: runs-on: ubuntu-latest @@ -125,3 +127,10 @@ jobs: helm template windrose ./helm/windrose >/dev/null helm template windrose ./helm/windrose --set serverConfig.mode=managed >/dev/null helm template windrose ./helm/windrose --set serverConfig.mode=mutable >/dev/null + helm template windrose ./helm/windrose \ + --set metrics.enabled=true \ + --set metrics.serviceMonitor.enabled=true \ + --set metrics.grafanaDashboard.enabled=true >/dev/null + ! helm template windrose ./helm/windrose \ + --set metrics.enabled=true \ + --set metrics.port=28080 >/dev/null diff --git a/Dockerfile b/Dockerfile index 65cf3a9..018c997 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,9 +70,10 @@ COPY --chmod=755 scripts/reconcile-engine-ini.sh /usr/local/bin/reconcile-engine # the same image as the game binary; the UI sidecar runs via a command # override at /opt/windrose-ui/server.py. Backend lives at the repo # root (server.py); frontend bundle is the sibling ui/ tree. -COPY --chown=10000:10000 server.py /opt/windrose-ui/server.py -COPY --chown=10000:10000 ui/ /opt/windrose-ui/ui/ -RUN chmod 755 /opt/windrose-ui/server.py +COPY --chown=10000:10000 server.py /opt/windrose-ui/server.py +COPY --chown=10000:10000 metrics.py /opt/windrose-ui/metrics.py +COPY --chown=10000:10000 ui/ /opt/windrose-ui/ui/ +RUN chmod 755 /opt/windrose-ui/server.py /opt/windrose-ui/metrics.py USER steam diff --git a/README.md b/README.md index 98ffefb..4f5f7f5 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ This is a community project. It is not affiliated with or endorsed by the Windro The Windrose dedicated-server Steam app (id `4129620`) pulls fine via anonymous SteamCMD — that's the default bootstrap, so a fresh pod / droplet / compose stack goes from nothing to a running server without any WindowsServer tarball work. Save data lives on persistent storage and survives game patches automatically. The admin console also exposes an upload path for operators running a pre-release or modded `WindowsServer/` build; see *Optional: Bring Your Own Server Files* below. -The pod runs three containers: +The pod runs three containers by default: - **`windrose`** — the game itself under GE-Proton. Only runs the game binary; no backgrounded work in its shell (Proton hates shell job-control races with Xvfb). - **`xvfb`** — a dedicated X display server on `:99`, shared into the game container via an `emptyDir` at `/tmp/.X11-unix`. Lives in its own container so its signal space can't interfere with Proton. - **`windrose-ui`** — a stdlib-only Python admin console (served from the same image as the game container via `python3 /opt/windrose-ui/server.py`). Exposes the invite-code card, server/players/resources status, config editor, backups, per-world editor, manual `WindowsServer` upload, and Discord/generic webhook dispatch. Shares the PVC with the game container so both see the same filesystem. -All three share the pod's PID namespace (`shareProcessNamespace: true`) so the UI sidecar can `pgrep` for the game process. +All three share the pod's PID namespace (`shareProcessNamespace: true`) so the UI sidecar can `pgrep` for the game process. Helm can also add an opt-in **`windrose-metrics`** sidecar for Prometheus scraping. Other Windrose dockerizations exist — this one leans on patterns we already operate in [`enshrouded-self-hosted`](https://github.com/shipstuff/enshrouded-self-hosted): GE-Proton, non-root container, PVC-backed persistence, host networking, Helm + plain manifests + Docker Compose in sync. @@ -140,15 +140,24 @@ for the full env list; it's commented inline. All three containers come up: `windrose` (game, `network_mode: host`), `xvfb` (display server), `windrose-ui` (UI on `127.0.0.1:28080` by default). +Enable the optional Prometheus exporter sidecar with the `metrics` compose profile: + +```bash +docker compose --profile metrics up -d +curl -s http://127.0.0.1:9464/metrics +``` + ## Install On Bare Linux ```bash sudo ./bare-linux/install.sh ``` -Three systemd system services (game + Xvfb + admin UI), running as a -non-root `steam` user. UI binds to `127.0.0.1` by default; override -with `UI_BIND=0.0.0.0 UI_PASSWORD=…` at install time. See +Three systemd system services (game + Xvfb + admin UI), plus an optional +Prometheus metrics service, running as a non-root `steam` user. UI binds +to `127.0.0.1` by default; override with +`UI_BIND=0.0.0.0 UI_PASSWORD=…` at install time. Enable the metrics +exporter with `WINDROSE_METRICS_ENABLED=true`. See [`bare-linux/README.md`](bare-linux/README.md) for sizing, swap recipe, and the pre-loaded-world workflow (recommended for small VPSes). @@ -205,6 +214,13 @@ Every variable below is consumed by the container entrypoint, so it applies iden | `UI_PASSWORD` | `` | HTTP basic-auth password; empty = no auth (only safe on LAN-only / firewalled hosts). Username is ignored. | | `UI_ENABLE_ADMIN_WITHOUT_PASSWORD` | `false` | Explicit opt-in for destructive endpoints when `UI_PASSWORD` is empty. With a password set, destructive is always allowed. | | `UI_SERVE_STATIC` | `true` | Set `false` to have the Python sidecar serve only `/api/*`; pair with an nginx that owns the static bundle. | +| `UI_ENABLE_METRICS_ROUTE` | `false` | Optional simple-install mode: expose Prometheus metrics at the admin UI's `/metrics` route. Kubernetes should prefer the dedicated metrics sidecar. | + +**Metrics exporter** +| Env var | Default | Purpose | +|---|---|---| +| `METRICS_BIND` | `0.0.0.0` | Bind address for standalone `python3 /opt/windrose-ui/metrics.py`. | +| `METRICS_PORT` | `9464` | Prometheus scrape port for the standalone exporter. | **Backups** | Env var | Default | Purpose | @@ -229,6 +245,34 @@ Every variable below is consumed by the container entrypoint, so it applies iden | `WINDROSE_WEBHOOK_POLL_SECONDS` | `15` | Poll cadence for the event detector thread. | | `WINDROSE_WEBHOOK_TIMEOUT` | `5` | HTTP POST timeout (seconds). | +## Prometheus Metrics And Grafana + +The metrics exporter is stdlib Python in [`metrics.py`](metrics.py). It can run as a separate process (`python3 /opt/windrose-ui/metrics.py`) or be imported by the admin server for an opt-in `/metrics` route. Kubernetes installs should use the dedicated sidecar so metrics stay isolated from the admin UI. + +![Grafana dashboard: Windrose server metrics](docs/screenshots/03-grafana-dashboard.jpg) + +Helm example: + +```yaml +metrics: + enabled: true + serviceAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "9464" + prometheus.io/job: windrose-canary + serviceMonitor: + enabled: false + grafanaDashboard: + enabled: true +``` + +Use either `metrics.serviceMonitor.*` for Prometheus Operator or `metrics.serviceAnnotations` for plain Prometheus annotation discovery. If your Prometheus install only selects `ServiceMonitor`s with a release label, set it under `metrics.serviceMonitor.labels`. If Grafana only watches a monitoring namespace for dashboard ConfigMaps, set `metrics.grafanaDashboard.namespace`. + +**Multiple servers.** Grafana does not discover Windrose servers directly; it asks Prometheus for `job` and `instance` label values on `windrose_exporter_scrape_success`. If Prometheus only scrapes canary, canary is the only option in the dashboard. Prometheus attaches target labels such as `job` and `instance` to every scrape, so the packaged dashboard can view all Windrose servers together or drill into one. For annotation-based Prometheus installs, set a unique `prometheus.io/job` per server/release so the dropdown is readable; for Compose or bare-Linux, use distinct Prometheus scrape jobs or relabel `instance` to a friendly server name. + +The exporter intentionally publishes aggregate operational state only: running status, player counts, process CPU/RSS/uptime, resource ceilings, staged config/world/mod changes, mod counts, backup counts, backend region, save version, and build identity from Steam/logs. It does not publish invite codes, player account IDs, or player names. + ## Update The Server On Game Patch When Windrose ships a patch, the dedicated-server binary bumps its `` save-path segment. Entrypoint migrates worlds forward automatically when it sees ≥2 version folders under `RocksDB/`. @@ -398,6 +442,7 @@ All routes are served by the `windrose-ui` container at `:28080`. Static assets | Method | Path | Auth | Purpose | | ------ | --------------------------------------------- | ------------- | --------------------------------------------------------------------------------------- | | GET | `/healthz` | open | Liveness — returns `ok`. Safe for k8s probes and external monitors. | +| GET | `/metrics` | open | Optional Prometheus scrape endpoint when `UI_ENABLE_METRICS_ROUTE=true`. Prefer the metrics sidecar on k8s. | | GET | `/`, `/app.css`, `/app.js`, `/index.html` | open | Served when `ui.serveStatic=true` (default). Disable if nginx serves the static assets. | | GET | `/api/status` | open / authed | Game process state, player list, resource usage, invite code, backend region, staged-change hints. Public view redacts `AccountId`s and omits `allowDestructive` / `stagedWorlds`. | | GET | `/api/invite` | authed | Plain-text invite code. | @@ -511,10 +556,11 @@ kubectl kustomize . >/dev/null helm lint ./helm/windrose helm template windrose ./helm/windrose >/dev/null shellcheck scripts/entrypoint.sh scripts/pack-windowsserver.sh -python3 -m py_compile server.py +python3 -m py_compile server.py metrics.py python3 tests/test_retention.py python3 tests/test_restore.py python3 tests/test_auto_backup.py +python3 tests/test_metrics.py python3 tests/test_http.py bash tests/test_api.sh # requires a running canary — see CLAUDE.md ``` diff --git a/bare-linux/README.md b/bare-linux/README.md index fe58f71..b46845e 100644 --- a/bare-linux/README.md +++ b/bare-linux/README.md @@ -1,8 +1,9 @@ # Bare Linux Install Run Windrose directly on a spare Linux box (Ubuntu 22.04+ / Debian 12+) as -three systemd system services: game + Xvfb + admin UI. Validated on an -Ubuntu 24.04 DigitalOcean droplet with 2 cores / 4 GiB RAM; should run +three systemd system services by default: game + Xvfb + admin UI. A fourth +Prometheus metrics exporter service is available as an opt-in. Validated on +an Ubuntu 24.04 DigitalOcean droplet with 2 cores / 4 GiB RAM; should run anywhere the [`Dockerfile`](../Dockerfile) deps are available. ## Sizing @@ -67,6 +68,9 @@ See § What The Install Includes for the full picture. - `windrose-xvfb.service` — virtual display on `:99` - `windrose-game.service` — the game under GE-Proton - `windrose-ui.service` — the Python admin console (stdlib, no deps) +- **Optional metrics unit**: + - `windrose-metrics.service` — Prometheus exporter on `127.0.0.1:9464` + when `WINDROSE_METRICS_ENABLED=true` - **Files under `/opt/windrose/`** (the installed code) and **`/home/steam/windrose/`** (the PVC-equivalent: game binaries, saves, backups). `/etc/windrose/windrose.env` holds all runtime knobs, @@ -98,6 +102,33 @@ sudo journalctl -fu windrose-game sudo journalctl -fu windrose-ui ``` +## Prometheus Metrics + +Enable the standalone metrics exporter at install time: + +```bash +sudo WINDROSE_METRICS_ENABLED=true ./bare-linux/install.sh +curl -s http://127.0.0.1:9464/metrics +``` + +The installer writes `windrose-metrics.service` on every run, but only +enables and starts it when `WINDROSE_METRICS_ENABLED=true`. Re-run with +`WINDROSE_METRICS_ENABLED=false` to disable and stop the service. + +Defaults are loopback-only: + +```env +WINDROSE_METRICS_ENABLED=false +METRICS_BIND=127.0.0.1 +METRICS_PORT=9464 +``` + +For a Prometheus running on the same host, scrape `127.0.0.1:9464`. For a +remote Prometheus, either use an SSH tunnel/reverse proxy or set +`METRICS_BIND=0.0.0.0` and firewall the port. The metrics endpoint has no +auth; it does not expose invite codes or player identities, but it does +expose operational state. + ## Overrides All env vars are read from `/etc/windrose/windrose.env`. The installer @@ -119,6 +150,10 @@ Or just edit the env file and `systemctl restart windrose-game` after. | `UI_PORT` | `28080` | UI listen port | | `UI_PASSWORD` | empty | HTTP basic-auth password. Strongly recommended for any publicly-reachable host. | | `UI_ENABLE_ADMIN_WITHOUT_PASSWORD` | `false` | explicit opt-in for destructive routes when no password is set — LAN-only | +| `UI_ENABLE_METRICS_ROUTE` | `false` | expose `/metrics` from `windrose-ui` instead of, or in addition to, the standalone metrics service | +| `WINDROSE_METRICS_ENABLED` | `false` | enable/start `windrose-metrics.service` | +| `METRICS_BIND` | `127.0.0.1` | metrics exporter listen iface | +| `METRICS_PORT` | `9464` | metrics exporter listen port | | `SERVER_NAME` | `Windrose Bare-Linux` | informational | | `MAX_PLAYER_COUNT` | `4` | 4 is the vendor guide; up to 10 with more RAM | | `WORLD_NAME` | `Default Windrose World` | display name | @@ -246,11 +281,12 @@ the game's RSS spikes on world load + backend handshake. ``` /opt/windrose/scripts/entrypoint.sh # game launcher /opt/windrose/server.py # admin console +/opt/windrose/metrics.py # Prometheus exporter /opt/windrose/ui/{index.html,app.js,app.css} /etc/windrose/windrose.env # runtime env (root-rw, group-r for steam) /home/steam/windrose/ # game data (WindowsServer/, saves, backups) /home/steam/steamcmd/ # SteamCMD + GE-Proton compat data -/etc/systemd/system/windrose-{xvfb,game,ui}.service +/etc/systemd/system/windrose-{xvfb,game,ui,metrics}.service ``` The admin console writes backups into `/home/steam/backups//` @@ -259,8 +295,8 @@ The admin console writes backups into `/home/steam/backups//` ## Uninstall ```bash -sudo systemctl disable --now windrose-game windrose-ui windrose-xvfb -sudo rm /etc/systemd/system/windrose-{game,ui,xvfb}.service +sudo systemctl disable --now windrose-game windrose-ui windrose-xvfb windrose-metrics +sudo rm /etc/systemd/system/windrose-{game,ui,xvfb,metrics}.service sudo systemctl daemon-reload # Data under /home/steam/ stays — delete manually if desired: # sudo userdel -r steam diff --git a/bare-linux/install.sh b/bare-linux/install.sh index e270145..4807882 100755 --- a/bare-linux/install.sh +++ b/bare-linux/install.sh @@ -1,6 +1,7 @@ #!/bin/bash -# Install the Windrose dedicated server as three systemd services -# (game + Xvfb + admin UI) on a bare Linux box. Tested on Ubuntu +# Install the Windrose dedicated server as three systemd services by default +# (game + Xvfb + admin UI) on a bare Linux box, with an optional metrics +# exporter service. Tested on Ubuntu # 24.04; should work on Debian 12+ / Ubuntu 22.04+ with no changes. # # Run from the repo root (or anywhere as long as the paths resolve): @@ -9,12 +10,16 @@ # Overrides: # WINDROSE_USER user that owns the install (default: steam) # WINDROSE_INSTALL_DIR where scripts/* land (default: /opt/windrose) -# UI_BIND UI listen interface (default: 0.0.0.0) +# UI_BIND UI listen interface (default: 127.0.0.1) # UI_PORT UI listen port (default: 28080) # UI_PASSWORD HTTP basic-auth password (default: empty) # UI_ENABLE_ADMIN_WITHOUT_PASSWORD # explicit opt-in for destructive routes when # UI_PASSWORD is empty (default: false) +# WINDROSE_METRICS_ENABLED install/start Prometheus exporter service +# (default: false) +# METRICS_BIND metrics listen interface (default: 127.0.0.1) +# METRICS_PORT metrics listen port (default: 9464) # WINDROSE_PATCH_IDLE_CPU opt in to the idle-CPU binary patch # (default: 0; flip to "1" to apply on boot) # SERVER_NAME, MAX_PLAYER_COUNT, WORLD_NAME, WORLD_PRESET_TYPE, @@ -23,8 +28,8 @@ # through to the entrypoint's default. # # Uninstall: -# sudo systemctl disable --now windrose-game windrose-ui windrose-xvfb -# sudo rm /etc/systemd/system/windrose-{game,ui,xvfb}.service +# sudo systemctl disable --now windrose-game windrose-ui windrose-xvfb windrose-metrics +# sudo rm /etc/systemd/system/windrose-{game,ui,xvfb,metrics}.service # sudo systemctl daemon-reload # (data under /home/steam/ stays — delete manually if desired) @@ -68,6 +73,17 @@ if [ "${UI_BIND:-127.0.0.1}" = "0.0.0.0" ] && [ -z "${UI_PASSWORD:-}" ] && [ "${ printf ' and reverse-proxy via nginx/caddy with auth in front.\033[0m\n' >&2 fi +case "${WINDROSE_METRICS_ENABLED:-false}" in + 1|true|TRUE|yes|YES) + if [ "${METRICS_BIND:-127.0.0.1}" = "0.0.0.0" ]; then + printf '\033[33m[install] WARN: METRICS_BIND=0.0.0.0 exposes unauthenticated\n' + printf ' Prometheus metrics. They do not include invite codes or\n' + printf ' player identities, but they do expose operational state.\n' + printf ' Prefer loopback + Prometheus on-host scrape or firewall it.\033[0m\n' >&2 + fi + ;; +esac + log() { printf '[install] %s\n' "$*"; } warn() { printf '\033[33m[install] WARN: %s\033[0m\n' "$*" >&2; } @@ -197,6 +213,8 @@ install -m 0644 "${SCRIPTS_SRC}/WorldDescription_example.json" \ install -d -o "${WINDROSE_USER}" -g "${WINDROSE_GROUP}" "${WINDROSE_INSTALL_DIR}/ui" install -m 0755 -o "${WINDROSE_USER}" -g "${WINDROSE_GROUP}" \ "${REPO_ROOT}/server.py" "${WINDROSE_INSTALL_DIR}/server.py" +install -m 0755 -o "${WINDROSE_USER}" -g "${WINDROSE_GROUP}" \ + "${REPO_ROOT}/metrics.py" "${WINDROSE_INSTALL_DIR}/metrics.py" for f in index.html app.js app.css; do install -m 0644 -o "${WINDROSE_USER}" -g "${WINDROSE_GROUP}" \ "${UI_SRC}/${f}" "${WINDROSE_INSTALL_DIR}/ui/${f}" @@ -268,6 +286,8 @@ _MANAGED_KEYS=" \ FILES_WAIT_TIMEOUT_SECONDS WINDROSE_PATCH_IDLE_CPU \ UI_BIND UI_PORT UI_PASSWORD \ UI_ENABLE_ADMIN_WITHOUT_PASSWORD UI_SERVE_STATIC \ + UI_ENABLE_METRICS_ROUTE WINDROSE_METRICS_ENABLED \ + METRICS_BIND METRICS_PORT \ WINDROSE_DISCORD_WEBHOOK_URL WINDROSE_WEBHOOK_URL \ WINDROSE_WEBHOOK_EVENTS WINDROSE_WEBHOOK_POLL_SECONDS \ WINDROSE_WEBHOOK_TIMEOUT \ @@ -321,6 +341,10 @@ fi : "${UI_PORT:=28080}" : "${UI_PASSWORD:=}" : "${UI_ENABLE_ADMIN_WITHOUT_PASSWORD:=false}" +: "${UI_ENABLE_METRICS_ROUTE:=false}" +: "${WINDROSE_METRICS_ENABLED:=false}" +: "${METRICS_BIND:=127.0.0.1}" +: "${METRICS_PORT:=9464}" log "writing env file ${WINDROSE_ENV_FILE}" tmp_env="$(mktemp)" @@ -367,6 +391,14 @@ UI_PORT=${UI_PORT:-28080} UI_PASSWORD=${UI_PASSWORD:-} UI_ENABLE_ADMIN_WITHOUT_PASSWORD=${UI_ENABLE_ADMIN_WITHOUT_PASSWORD:-false} UI_SERVE_STATIC=${UI_SERVE_STATIC:-true} +UI_ENABLE_METRICS_ROUTE=${UI_ENABLE_METRICS_ROUTE:-false} + +# Prometheus metrics. WINDROSE_METRICS_ENABLED controls the standalone +# windrose-metrics.service; UI_ENABLE_METRICS_ROUTE exposes the same +# payload from windrose-ui at /metrics for simpler reverse-proxy setups. +WINDROSE_METRICS_ENABLED=${WINDROSE_METRICS_ENABLED:-false} +METRICS_BIND=${METRICS_BIND:-127.0.0.1} +METRICS_PORT=${METRICS_PORT:-9464} # Webhook notifications — Discord embed + generic JSON POST. Leave URLs # empty to disable delivery (the EventDetector thread still runs but @@ -473,9 +505,36 @@ RestartSec=5 WantedBy=multi-user.target " +write_unit "windrose-metrics.service" "[Unit] +Description=Windrose Prometheus Metrics Exporter +After=network-online.target windrose-game.service +Wants=network-online.target + +[Service] +Type=simple +User=${WINDROSE_USER} +Group=${WINDROSE_GROUP} +WorkingDirectory=${WINDROSE_HOME} +EnvironmentFile=${WINDROSE_ENV_FILE} +ExecStart=/usr/bin/python3 ${WINDROSE_INSTALL_DIR}/metrics.py +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target +" + # --- Reload + enable -------------------------------------------------- systemctl daemon-reload systemctl enable --now windrose-xvfb.service windrose-ui.service windrose-game.service +case "${WINDROSE_METRICS_ENABLED}" in + 1|true|TRUE|yes|YES) + systemctl enable --now windrose-metrics.service + ;; + *) + systemctl disable --now windrose-metrics.service >/dev/null 2>&1 || true + ;; +esac # `enable --now` is a no-op on services that are already running, so # re-runs of install.sh (e.g. picking up new UI code) wouldn't restart # them — the Python process would keep the old server.py in memory. @@ -483,15 +542,26 @@ systemctl enable --now windrose-xvfb.service windrose-ui.service windrose-game.s # fresh installs aren't double-started and upgrades actually pick up # new code without the operator having to chase extra systemctl calls. systemctl try-restart windrose-ui.service windrose-game.service +case "${WINDROSE_METRICS_ENABLED}" in + 1|true|TRUE|yes|YES) + systemctl try-restart windrose-metrics.service + ;; +esac log "done." echo echo " Services run as: ${WINDROSE_USER} (non-root; systemd units at" -echo " /etc/systemd/system/windrose-{xvfb,game,ui}.service)" +echo " /etc/systemd/system/windrose-{xvfb,game,ui,metrics}.service)" echo " Game data lives: ${WINDROSE_HOME}/windrose/" echo " Env file (edit): ${WINDROSE_ENV_FILE}" echo " Tail game logs: sudo journalctl -fu windrose-game" echo " Tail UI logs: sudo journalctl -fu windrose-ui" +case "${WINDROSE_METRICS_ENABLED}" in + 1|true|TRUE|yes|YES) + echo " Metrics: http://${METRICS_BIND}:${METRICS_PORT}/metrics" + echo " Tail metrics logs: sudo journalctl -fu windrose-metrics" + ;; +esac echo echo " Admin console: http://${UI_BIND}:${UI_PORT}/" if [ "${UI_BIND}" = "127.0.0.1" ]; then diff --git a/docker-compose.yaml b/docker-compose.yaml index 1aa3c9e..3dbdd2f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -86,6 +86,8 @@ services: # Opt-in: allow destructive actions when no password is set (LAN-only). # With UI_PASSWORD set, destructive is always allowed. UI_ENABLE_ADMIN_WITHOUT_PASSWORD: ${UI_ENABLE_ADMIN_WITHOUT_PASSWORD:-false} + # Optional simple-install mode. Prefer windrose-metrics for Prometheus. + UI_ENABLE_METRICS_ROUTE: ${UI_ENABLE_METRICS_ROUTE:-false} # Optional Discord / generic JSON webhook notifications. Background # thread polls game state every WINDROSE_WEBHOOK_POLL_SECONDS and # fires events (server.online/offline, player.join/leave, plus @@ -104,6 +106,24 @@ services: volumes: - windrose-data:/home/steam + windrose-metrics: + profiles: ["metrics"] + image: ${WINDROSE_IMAGE:-ghcr.io/shipstuff/windrose-server:0.3.0} + restart: unless-stopped + pid: service:windrose + depends_on: + - windrose + entrypoint: ["python3", "/opt/windrose-ui/metrics.py"] + environment: + METRICS_BIND: "0.0.0.0" + METRICS_PORT: ${METRICS_PORT:-9464} + WINDROSE_GAME_CPU_LIMIT: ${WINDROSE_GAME_CPU_LIMIT:-} + WINDROSE_GAME_MEM_LIMIT: ${WINDROSE_GAME_MEM_LIMIT:-} + ports: + - "${METRICS_BIND:-127.0.0.1}:${METRICS_PORT:-9464}:${METRICS_PORT:-9464}" + volumes: + - windrose-data:/home/steam + volumes: windrose-data: x11-socket: diff --git a/docs/screenshots/03-grafana-dashboard.jpg b/docs/screenshots/03-grafana-dashboard.jpg new file mode 100755 index 0000000..16b0c19 Binary files /dev/null and b/docs/screenshots/03-grafana-dashboard.jpg differ diff --git a/helm/windrose/README.md b/helm/windrose/README.md index 81c5ac6..c81aaf2 100644 --- a/helm/windrose/README.md +++ b/helm/windrose/README.md @@ -162,6 +162,53 @@ ui: `urlSecret` / `discordUrlSecret` take precedence over `url` / `discordUrl` when both are set. +## Prometheus Metrics + Grafana Dashboard + +The chart can add an opt-in `windrose-metrics` sidecar that serves +Prometheus text metrics from the same PVC and shared PID namespace as the +admin UI. This keeps monitoring code out of the admin surface while still +letting the exporter reuse the same stdlib filesystem/process parsers. + +```yaml +metrics: + enabled: true + # Plain Prometheus annotation discovery: + serviceAnnotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "9464" + prometheus.io/job: windrose-canary + # Prometheus Operator: + serviceMonitor: + enabled: false + # labels: + # release: kube-prometheus-stack + grafanaDashboard: + enabled: true + # namespace: monitoring +``` + +Use `metrics.serviceAnnotations` for plain Prometheus annotation +discovery, or set `metrics.serviceMonitor.enabled` for Prometheus +Operator. Set `metrics.serviceMonitor.labels` if your Prometheus operator +only selects monitors with a specific release label. The dashboard is +packaged as a ConfigMap using `metrics.grafanaDashboard.labels`; set +`metrics.grafanaDashboard.namespace` for Grafana sidecars that only watch +their own namespace. + +The packaged dashboard includes `Server job` and `Target instance` +variables populated from Prometheus labels on +`windrose_exporter_scrape_success`. Grafana does not discover Windrose +servers directly; if Prometheus only scrapes canary, canary is the only +dashboard option. For multi-server operators, give each Windrose release a +unique Prometheus job label when using annotation scraping, or rely on your +Prometheus/ServiceMonitor relabeling to provide readable `job` and +`instance` labels. + +For simpler non-operator installs, `metrics.uiRouteEnabled: true` exposes +the same scrape payload at the admin server's `/metrics` route. Kubernetes +deployments should prefer the sidecar. + ## Evict A Flaky Backend Region Windrose's gRPC gateways in KR / EU / RU occasionally hit 502s and @@ -259,6 +306,24 @@ the chart is a thin wrapper around those vars plus Kubernetes-level knobs | `ingress.annotations` | nginx tuning for 8 GiB upload cap, 1h upstream timeout, streaming bodies | Sized for the UI upload + saves-download endpoints. | | `ingress.tls` | `[]` | Standard k8s TLS block. | +### Metrics + +| Value | Default | Purpose | +|---|---|---| +| `metrics.enabled` | `false` | Add a dedicated `windrose-metrics` sidecar and service port. | +| `metrics.port` | `9464` | Port exposed by the metrics sidecar and service. Must differ from `service.port` when `hostNetwork: true`. | +| `metrics.serviceAnnotations` | `{}` | Extra Service annotations, useful for plain Prometheus `prometheus.io/*` annotation scraping. | +| `metrics.uiRouteEnabled` | `false` | Also expose `/metrics` from the admin UI process. Useful for small/simple installs; the sidecar is preferred on k8s. | +| `metrics.serviceMonitor.enabled` | `false` | Render a Prometheus Operator `ServiceMonitor` for the metrics service port. | +| `metrics.serviceMonitor.interval` | `30s` | Prometheus scrape interval. | +| `metrics.serviceMonitor.scrapeTimeout` | `10s` | Prometheus scrape timeout. | +| `metrics.serviceMonitor.labels` | `{}` | Extra labels for Prometheus selector compatibility. | +| `metrics.serviceMonitor.annotations` | `{}` | Extra annotations on the ServiceMonitor. | +| `metrics.grafanaDashboard.enabled` | `false` | Render the packaged Windrose Grafana dashboard ConfigMap. | +| `metrics.grafanaDashboard.namespace` | `""` | Namespace for the dashboard ConfigMap. Empty means the Windrose release namespace. | +| `metrics.grafanaDashboard.labels` | `grafana_dashboard: "1"` | Dashboard ConfigMap labels for Grafana sidecar discovery. | +| `metrics.grafanaDashboard.annotations` | `{}` | Extra annotations on the dashboard ConfigMap. | + ### Game runtime | Value | Default | Purpose | @@ -306,6 +371,7 @@ the chart is a thin wrapper around those vars plus Kubernetes-level knobs | `resources.game.limits.cpu` | *(unset)* | Leave unset to let the patch pace the idle loop; set a cap (e.g. `500m`) as belt-and-braces. | | `resources.xvfb.{requests,limits}` | `cpu: 50m`/`memory: 32Mi` → `memory: 256Mi` | Xvfb is tiny. | | `resources.ui.{requests,limits}` | `cpu: 10m`/`memory: 32Mi` → `memory: 256Mi` | 256Mi covers `/api/idle-cpu-patch` scanning an unknown-MD5 binary; steady state is ~20 MB. | +| `resources.metrics.{requests,limits}` | `cpu: 10m`/`memory: 32Mi` → `memory: 128Mi` | Prometheus exporter sidecar. | ### Security diff --git a/helm/windrose/dashboards/windrose-overview.json b/helm/windrose/dashboards/windrose-overview.json new file mode 100644 index 0000000..1355b8d --- /dev/null +++ b/helm/windrose/dashboards/windrose-overview.json @@ -0,0 +1,706 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "text": "Down" + }, + "1": { + "text": "Running" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "expr": "windrose_server_running{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Server", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "windrose_players_current{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Players", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "windrose_process_cpu_percent{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Game CPU", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "windrose_process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Game RSS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "windrose_process_uptime_seconds{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "windrose_worlds_total{job=~\"$job\",instance=~\"$instance\"}", + "refId": "A", + "legendFormat": "{{job}} {{instance}}" + } + ], + "title": "Worlds", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "windrose_process_cpu_percent{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "windrose_process_resident_memory_bytes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{job}} {{instance}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "windrose_staged_changes{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{job}} {{instance}} {{type}}", + "refId": "A" + } + ], + "title": "Staged Changes", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 10, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "windrose_mods_total{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{job}} {{instance}} {{state}}", + "refId": "A" + } + ], + "title": "Mods", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "windrose_backups_total{job=~\"$job\",instance=~\"$instance\"}", + "legendFormat": "{{job}} {{instance}} {{source}}", + "refId": "A" + } + ], + "title": "Backups", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "displayMode": "color-background" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 12, + "options": { + "showHeader": true + }, + "targets": [ + { + "expr": "windrose_game_build_info{job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "A", + "legendFormat": "{{job}} {{instance}} build" + }, + { + "expr": "windrose_backend_region_info{job=~\"$job\",instance=~\"$instance\"}", + "format": "table", + "instant": true, + "refId": "B", + "legendFormat": "{{job}} {{instance}} backend" + } + ], + "title": "Build / Backend", + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "windrose" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(windrose_exporter_scrape_success, job)", + "hide": 0, + "includeAll": true, + "label": "Server job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(windrose_exporter_scrape_success, job)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(windrose_exporter_scrape_success{job=~\"$job\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Target instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(windrose_exporter_scrape_success{job=~\"$job\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Windrose Server", + "uid": "windrose-server", + "version": 1, + "weekStart": "" +} diff --git a/helm/windrose/templates/grafana-dashboard-configmap.yaml.tpl b/helm/windrose/templates/grafana-dashboard-configmap.yaml.tpl new file mode 100644 index 0000000..0bf9274 --- /dev/null +++ b/helm/windrose/templates/grafana-dashboard-configmap.yaml.tpl @@ -0,0 +1,19 @@ +{{- if .Values.metrics.grafanaDashboard.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "windrose.fullname" . }}-grafana-dashboard + namespace: {{ .Values.metrics.grafanaDashboard.namespace | default .Values.namespace }} + labels: +{{ include "windrose.labels" . | indent 4 }} +{{- with .Values.metrics.grafanaDashboard.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- with .Values.metrics.grafanaDashboard.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +data: + windrose-overview.json: |- +{{ .Files.Get "dashboards/windrose-overview.json" | indent 4 }} +{{- end }} diff --git a/helm/windrose/templates/service.yaml.tpl b/helm/windrose/templates/service.yaml.tpl index 472ea34..74ef8f0 100644 --- a/helm/windrose/templates/service.yaml.tpl +++ b/helm/windrose/templates/service.yaml.tpl @@ -5,6 +5,12 @@ metadata: namespace: {{ .Values.namespace }} labels: {{ include "windrose.labels" . | indent 4 }} +{{- with .Values.metrics.serviceAnnotations }} + annotations: +{{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} +{{- end }} +{{- end }} spec: type: {{ .Values.service.type }} selector: @@ -18,3 +24,9 @@ spec: protocol: TCP port: {{ .Values.service.port }} targetPort: ui + {{- if .Values.metrics.enabled }} + - name: metrics + protocol: TCP + port: {{ .Values.metrics.port }} + targetPort: metrics + {{- end }} diff --git a/helm/windrose/templates/servicemonitor.yaml.tpl b/helm/windrose/templates/servicemonitor.yaml.tpl new file mode 100644 index 0000000..05f4883 --- /dev/null +++ b/helm/windrose/templates/servicemonitor.yaml.tpl @@ -0,0 +1,26 @@ +{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "windrose.fullname" . }} + namespace: {{ .Values.namespace }} + labels: +{{ include "windrose.labels" . | indent 4 }} +{{- with .Values.metrics.serviceMonitor.labels }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- with .Values.metrics.serviceMonitor.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ include "windrose.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + endpoints: + - port: metrics + path: /metrics + interval: {{ .Values.metrics.serviceMonitor.interval | quote }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout | quote }} +{{- end }} diff --git a/helm/windrose/templates/statefulset.yaml.tpl b/helm/windrose/templates/statefulset.yaml.tpl index ce1eb50..8449da8 100644 --- a/helm/windrose/templates/statefulset.yaml.tpl +++ b/helm/windrose/templates/statefulset.yaml.tpl @@ -1,3 +1,6 @@ +{{- if and .Values.hostNetwork .Values.metrics.enabled (eq (toString .Values.metrics.port) (toString .Values.service.port)) -}} +{{- fail "metrics.port must differ from service.port when hostNetwork=true and metrics.enabled=true" -}} +{{- end -}} apiVersion: apps/v1 kind: StatefulSet metadata: @@ -216,6 +219,8 @@ spec: value: {{ .Values.ui.enableAdminWithoutPassword | quote }} - name: UI_SERVE_STATIC value: {{ .Values.ui.serveStatic | quote }} + - name: UI_ENABLE_METRICS_ROUTE + value: {{ .Values.metrics.uiRouteEnabled | quote }} {{- with .Values.ui.webhooks }} - name: WINDROSE_WEBHOOK_EVENTS value: {{ .events | quote }} @@ -261,6 +266,33 @@ spec: - name: data mountPath: /home/steam subPath: {{ .Values.persistence.subPath | quote }} + {{- if .Values.metrics.enabled }} + - name: windrose-metrics + image: "{{ .Values.uiImage.repository | default .Values.image.repository }}:{{ .Values.uiImage.tag | default .Values.image.tag }}" + imagePullPolicy: {{ .Values.uiImage.pullPolicy | default .Values.image.pullPolicy }} + command: ["python3", "/opt/windrose-ui/metrics.py"] + securityContext: +{{ toYaml .Values.containerSecurityContext | indent 12 }} + env: + - name: METRICS_BIND + value: "0.0.0.0" + - name: METRICS_PORT + value: {{ .Values.metrics.port | quote }} + - name: WINDROSE_GAME_CPU_LIMIT + value: {{ .Values.resources.game.limits.cpu | default "" | quote }} + - name: WINDROSE_GAME_MEM_LIMIT + value: {{ .Values.resources.game.limits.memory | default "" | quote }} + ports: + - name: metrics + containerPort: {{ .Values.metrics.port }} + protocol: TCP + resources: +{{ toYaml .Values.resources.metrics | indent 12 }} + volumeMounts: + - name: data + mountPath: /home/steam + subPath: {{ .Values.persistence.subPath | quote }} + {{- end }} volumes: - name: data persistentVolumeClaim: diff --git a/helm/windrose/values.yaml b/helm/windrose/values.yaml index cf6dc2f..47a0773 100644 --- a/helm/windrose/values.yaml +++ b/helm/windrose/values.yaml @@ -73,6 +73,32 @@ ui: name: "" key: discord-webhook-url +# Optional Prometheus exporter. Kubernetes deployments should prefer this +# dedicated sidecar so metrics are isolated from the admin UI surface. +metrics: + enabled: false + port: 9464 + # For plain Prometheus setups that scrape services by annotation, e.g. + # prometheus.io/scrape: "true". Leave empty when using ServiceMonitor. + serviceAnnotations: {} + # Simple installs can expose /metrics from the admin UI process instead + # of running the sidecar. Helm keeps this false by default. + uiRouteEnabled: false + serviceMonitor: + enabled: false + interval: 30s + scrapeTimeout: 10s + labels: {} + annotations: {} + grafanaDashboard: + enabled: false + # Empty = same namespace as the Windrose release. Set to "monitoring" + # for Grafana sidecars that only watch their own namespace. + namespace: "" + labels: + grafana_dashboard: "1" + annotations: {} + persistence: existingClaim: "" size: 20Gi @@ -301,6 +327,12 @@ resources: memory: 32Mi limits: memory: 256Mi + metrics: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi securityContext: runAsUser: 10000 diff --git a/metrics.py b/metrics.py new file mode 100644 index 0000000..ce1a222 --- /dev/null +++ b/metrics.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Prometheus exporter for Windrose self-hosted. + +This file is intentionally separate from the admin console. It can run as +its own stdlib HTTP server (`python3 /opt/windrose-ui/metrics.py`) or be +imported by server.py for an optional /metrics route on simpler installs. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import time +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +import server + +BIND = os.environ.get("METRICS_BIND", "0.0.0.0") +PORT = int(os.environ.get("METRICS_PORT", "9464")) +_EXE_MD5_CACHE: tuple[Path, int, int, str] | None = None + + +def _label_value(value: Any) -> str: + text = "" if value is None else str(value) + return text.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def _labels(labels: dict[str, Any] | None) -> str: + if not labels: + return "" + parts = [f'{k}="{_label_value(v)}"' for k, v in sorted(labels.items())] + return "{" + ",".join(parts) + "}" + + +def _num(value: Any) -> str: + if isinstance(value, bool): + return "1" if value else "0" + if value is None: + return "0" + try: + return str(float(value)) + except (TypeError, ValueError): + return "0" + + +def _emit(out: list[str], seen: set[str], name: str, help_text: str, + metric_type: str, value: Any, labels: dict[str, Any] | None = None) -> None: + if name not in seen: + out.append(f"# HELP {name} {help_text}") + out.append(f"# TYPE {name} {metric_type}") + seen.add(name) + out.append(f"{name}{_labels(labels)} {_num(value)}") + + +def _read_tail(path: Path, limit: int = 262_144) -> str: + try: + size = path.stat().st_size + with path.open("rb") as f: + if size > limit: + f.seek(size - limit) + return f.read().decode("utf-8", errors="replace") + except OSError: + return "" + + +def _steam_build_id() -> str: + manifest = server.WINDROSE_SERVER_DIR / "steamapps" / "appmanifest_4129620.acf" + text = server.read_file(manifest) or "" + match = re.search(r'^\s*"buildid"\s*"([0-9]+)"', text, re.MULTILINE) + return match.group(1) if match else "" + + +def _exe_md5() -> str: + global _EXE_MD5_CACHE + exe = server.GAME_EXE_PATH + try: + st = exe.stat() + if ( + _EXE_MD5_CACHE + and _EXE_MD5_CACHE[0] == exe + and _EXE_MD5_CACHE[1] == st.st_mtime_ns + and _EXE_MD5_CACHE[2] == st.st_size + ): + return _EXE_MD5_CACHE[3] + h = hashlib.md5() # noqa: S324 - non-security fingerprint for build identity. + with exe.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + digest = h.hexdigest() + _EXE_MD5_CACHE = (exe, st.st_mtime_ns, st.st_size, digest) + return digest + except OSError: + _EXE_MD5_CACHE = None + return "" + + +def _game_build_from_log() -> dict[str, str]: + text = _read_tail(server.R5_LOG) + latest = {"gameVersion": "", "shaVersion": "", "releaseVersion": "", "deploymentId": ""} + pattern = re.compile( + r"GameVersion\s+(?P\S+?)\.\s+" + r"ShaVersion\s+(?P[0-9a-fA-F]+)\.\s+" + r"ReleaseVersion\s+(?P\S+?)\..*?" + r"DeploymentId\s+(?P\S+?)(?:\.|\s)" + ) + for match in pattern.finditer(text): + latest = { + "gameVersion": match.group("game"), + "shaVersion": match.group("sha"), + "releaseVersion": match.group("release"), + "deploymentId": match.group("deployment"), + } + return latest + + +def _collect_core(out: list[str], seen: set[str]) -> None: + cfg = server.load_json(server.CONFIG_PATH) or {} + persistent = cfg.get("ServerDescription_Persistent", {}) or cfg + pid, rss = server.find_game_pid() + players = server.parse_active_players() if pid else [] + worlds = server.find_worlds() + ceiling = server.resource_ceiling() + build = _game_build_from_log() + backend = server.backend_region() + + files_present = any([ + (server.WINDROSE_SERVER_DIR / "WindroseServer.exe").is_file(), + (server.WINDROSE_SERVER_DIR / "R5/Binaries/Win64/WindroseServer-Win64-Shipping.exe").is_file(), + ]) + + _emit(out, seen, "windrose_server_running", "Whether the Windrose game process is running.", "gauge", pid is not None) + _emit(out, seen, "windrose_server_files_present", "Whether WindowsServer files are present on disk.", "gauge", files_present) + _emit(out, seen, "windrose_server_password_protected", "Whether the game server is password protected.", "gauge", bool(persistent.get("IsPasswordProtected"))) + _emit(out, seen, "windrose_players_current", "Current connected player count.", "gauge", len(players)) + _emit(out, seen, "windrose_players_max", "Configured maximum player count.", "gauge", persistent.get("MaxPlayerCount", 0)) + _emit(out, seen, "windrose_worlds_total", "Number of discovered world directories.", "gauge", len(worlds)) + _emit(out, seen, "windrose_process_uptime_seconds", "Windrose game process uptime in seconds.", "gauge", server.game_uptime_seconds(pid)) + _emit(out, seen, "windrose_process_resident_memory_bytes", "Windrose game process resident memory in bytes.", "gauge", rss) + _emit(out, seen, "windrose_process_cpu_percent", "Windrose game process CPU percent, where 100 is one full CPU core.", "gauge", server.cpu_sample(pid)) + _emit(out, seen, "windrose_cpu_limit_millicores", "Detected game CPU limit in millicores.", "gauge", ceiling.get("cpuLimitMcpu", 0), {"source": ceiling.get("cpuLimitSource", "")}) + _emit(out, seen, "windrose_memory_limit_bytes", "Detected game memory limit in bytes.", "gauge", ceiling.get("memLimitBytes", 0), {"source": ceiling.get("memLimitSource", "")}) + _emit(out, seen, "windrose_staged_changes", "Pending staged changes by type.", "gauge", server.STAGED_CONFIG_PATH.is_file(), {"type": "server_config"}) + _emit(out, seen, "windrose_staged_changes", "Pending staged changes by type.", "gauge", sum(1 for w in worlds if w.get("staged")), {"type": "world"}) + _emit(out, seen, "windrose_staged_changes", "Pending staged changes by type.", "gauge", server.mods_staged_metadata_path().is_file(), {"type": "mods"}) + _emit(out, seen, "windrose_maintenance_mode", "Whether maintenance mode is active.", "gauge", server.MAINTENANCE_FLAG_FILE.is_file()) + _emit(out, seen, "windrose_backend_region_info", "Last backend gateway region observed in game logs.", "gauge", 1, {"region": backend}) + _emit(out, seen, "windrose_save_version_info", "Current save version discovered on disk.", "gauge", 1, {"version": server.current_save_version()}) + _emit(out, seen, "windrose_game_build_info", "Windrose game build identity observed from Steam and game logs.", "gauge", 1, { + "game_version": build.get("gameVersion", ""), + "sha_version": build.get("shaVersion", ""), + "release_version": build.get("releaseVersion", ""), + "deployment_id": build.get("deploymentId", ""), + "steam_buildid": _steam_build_id(), + "exe_md5": _exe_md5(), + }) + + +def _collect_mods(out: list[str], seen: set[str]) -> None: + state = server.list_mods_state() + mods = state.get("mods", []) + enabled = sum(1 for m in mods if m.get("enabled", True) and not m.get("pendingAction") == "delete") + disabled = sum(1 for m in mods if not m.get("enabled", True) and not m.get("pendingAction") == "delete") + pending = sum(1 for m in mods if m.get("pendingAction")) + _emit(out, seen, "windrose_mods_total", "Mods by state.", "gauge", enabled, {"state": "enabled"}) + _emit(out, seen, "windrose_mods_total", "Mods by state.", "gauge", disabled, {"state": "disabled"}) + _emit(out, seen, "windrose_mods_total", "Mods by state.", "gauge", pending, {"state": "pending"}) + _emit(out, seen, "windrose_mods_staged", "Whether staged mod metadata exists.", "gauge", bool(state.get("staged"))) + + +def _collect_backups(out: list[str], seen: set[str]) -> None: + newest_by_source: dict[str, dict[str, Any]] = {} + counts: dict[str, int] = {} + if server.BACKUP_ROOT.exists(): + for backup_dir in server.BACKUP_ROOT.iterdir(): + try: + if not backup_dir.is_dir(): + continue + st = backup_dir.stat() + except OSError: + continue + pinned = backup_dir.name.startswith(server.BACKUP_PIN_PREFIX) + auto = (backup_dir / server.AUTO_BACKUP_MARKER_NAME).is_file() + source = "auto" if auto else ("manual-pinned" if pinned else "manual") + counts[source] = counts.get(source, 0) + 1 + if source not in newest_by_source or st.st_mtime > newest_by_source[source]["mtime"]: + newest_by_source[source] = {"mtime": st.st_mtime} + for source, count in sorted(counts.items()): + _emit(out, seen, "windrose_backups_total", "Backups by source.", "gauge", count, {"source": source}) + for source, backup in sorted(newest_by_source.items()): + _emit(out, seen, "windrose_backup_latest_timestamp_seconds", "Unix timestamp of latest backup by source.", "gauge", backup["mtime"], {"source": source}) + + +def render_metrics() -> str: + start = time.monotonic() + out: list[str] = [] + seen: set[str] = set() + errors: list[str] = [] + + for collector in (_collect_core, _collect_mods, _collect_backups): + try: + collector(out, seen) + except Exception as exc: # noqa: BLE001 - exporter must stay scrapeable. + errors.append(f"{collector.__name__}: {exc}") + + _emit(out, seen, "windrose_exporter_scrape_success", "Whether the exporter completed all collectors.", "gauge", 0 if errors else 1) + _emit(out, seen, "windrose_exporter_scrape_duration_seconds", "Exporter scrape duration in seconds.", "gauge", round(time.monotonic() - start, 6)) + for idx, err in enumerate(errors): + _emit(out, seen, "windrose_exporter_collector_error", "Collector errors from the last scrape.", "gauge", 1, {"index": idx, "error": err[:160]}) + return "\n".join(out) + "\n" + + +class MetricsHandler(BaseHTTPRequestHandler): + server_version = "WindroseMetrics/1.0" + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API. + path = self.path.split("?", 1)[0] + if path == "/healthz": + self._send(HTTPStatus.OK, "text/plain; charset=utf-8", b"ok\n") + return + if path == "/metrics": + body = render_metrics().encode("utf-8") + self._send(HTTPStatus.OK, "text/plain; version=0.0.4; charset=utf-8", body) + return + self._send(HTTPStatus.NOT_FOUND, "text/plain; charset=utf-8", b"not found\n") + + def log_message(self, fmt: str, *args: Any) -> None: + return + + def _send(self, status: int, content_type: str, body: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main() -> None: + httpd = ThreadingHTTPServer((BIND, PORT), MetricsHandler) + print(f"windrose metrics exporter on {BIND}:{PORT}", flush=True) + print(f" windrose dir: {server.WINDROSE_SERVER_DIR}", flush=True) + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + finally: + httpd.server_close() + + +if __name__ == "__main__": + main() diff --git a/server.py b/server.py index a0acb8f..ff8a31d 100644 --- a/server.py +++ b/server.py @@ -8,6 +8,7 @@ Routing: GET / index.html GET /healthz (always open, no auth) + GET /metrics Prometheus metrics (only with UI_ENABLE_METRICS_ROUTE=true) GET /api/status full status JSON (status of game, players, resources) GET /api/invite plain-text invite code POST /api/upload stream tarball to PVC, preserve identity+saves @@ -86,6 +87,7 @@ # pod "/api/*-only" so an nginx in front (ingress or sidecar) can own # the static assets (and possibly auth) and reverse-proxy /api/* here. UI_SERVE_STATIC = os.environ.get("UI_SERVE_STATIC", "true").lower() not in ("0", "false", "no") +UI_ENABLE_METRICS_ROUTE = os.environ.get("UI_ENABLE_METRICS_ROUTE", "false").lower() in ("1", "true", "yes") BACKUP_RETAIN = int(os.environ.get("WINDROSE_BACKUP_RETAIN", "10")) BACKUP_RETAIN_DAYS = float(os.environ.get("WINDROSE_BACKUP_RETAIN_DAYS", "7")) # Auto-backup scheduler defaults. Zero on either disables that trigger. @@ -2265,6 +2267,12 @@ def do_DELETE(self): def _dispatch(self, method: str): path = urllib.parse.urlparse(self.path).path + if method == "GET" and path == "/metrics": + if UI_ENABLE_METRICS_ROUTE: + self._metrics() + else: + self._send(HTTPStatus.NOT_FOUND, "text/plain", b"not found\n") + return self._authed = check_basic_auth(self.headers.get("Authorization", "")) if path not in self.PUBLIC_PATHS and not self._authed: self.send_response(HTTPStatus.UNAUTHORIZED) @@ -2359,6 +2367,15 @@ def _forbidden(self, reason: str = "destructive operations are disabled"): def _healthz(self): self._send(HTTPStatus.OK, "text/plain", b"ok\n") + def _metrics(self): + try: + import metrics + except Exception as e: # noqa: BLE001 - surface import failure to scraper. + self._send(HTTPStatus.INTERNAL_SERVER_ERROR, "text/plain", f"metrics unavailable: {e}\n".encode()) + return + self._send(HTTPStatus.OK, "text/plain; version=0.0.4; charset=utf-8", + metrics.render_metrics().encode("utf-8")) + def _index(self): self._static("/index.html") diff --git a/tests/test_api.sh b/tests/test_api.sh index cb9c725..f468a51 100755 --- a/tests/test_api.sh +++ b/tests/test_api.sh @@ -6,7 +6,7 @@ # UI_AUTH=admin:canary — basic-auth creds; empty = unauth (then auth'd routes expect 401) # RUN_STOP=1 — also POST /api/server/stop (disruptive, off by default) # -# Default target: windrose-canary-0 / games / 28081 +# Default target: windrose-canary-0 / games / 28081 (canary admin UI) set -u POD="${1:-windrose-canary-0}" diff --git a/tests/test_install_env_merge.sh b/tests/test_install_env_merge.sh index 5e1cc73..234bf67 100755 --- a/tests/test_install_env_merge.sh +++ b/tests/test_install_env_merge.sh @@ -26,6 +26,10 @@ UI_PORT=28080 UI_PASSWORD=s3cretOperatorValue UI_ENABLE_ADMIN_WITHOUT_PASSWORD=false UI_SERVE_STATIC=true +UI_ENABLE_METRICS_ROUTE=false +WINDROSE_METRICS_ENABLED=true +METRICS_BIND=127.0.0.1 +METRICS_PORT=9464 WINDROSE_DISCORD_WEBHOOK_URL=https://discord.example/webhooks/abc WINDROSE_WEBHOOK_URL= WINDROSE_WEBHOOK_EVENTS=server.online,server.offline @@ -63,6 +67,8 @@ _MANAGED_KEYS=" \ FILES_WAIT_TIMEOUT_SECONDS WINDROSE_PATCH_IDLE_CPU \ UI_BIND UI_PORT UI_PASSWORD \ UI_ENABLE_ADMIN_WITHOUT_PASSWORD UI_SERVE_STATIC \ + UI_ENABLE_METRICS_ROUTE WINDROSE_METRICS_ENABLED \ + METRICS_BIND METRICS_PORT \ WINDROSE_DISCORD_WEBHOOK_URL WINDROSE_WEBHOOK_URL \ WINDROSE_WEBHOOK_EVENTS WINDROSE_WEBHOOK_POLL_SECONDS \ WINDROSE_WEBHOOK_TIMEOUT \ @@ -101,6 +107,10 @@ UI_PORT=\${UI_PORT:-28080} UI_PASSWORD=\${UI_PASSWORD:-} UI_ENABLE_ADMIN_WITHOUT_PASSWORD=\${UI_ENABLE_ADMIN_WITHOUT_PASSWORD:-false} UI_SERVE_STATIC=\${UI_SERVE_STATIC:-true} +UI_ENABLE_METRICS_ROUTE=\${UI_ENABLE_METRICS_ROUTE:-false} +WINDROSE_METRICS_ENABLED=\${WINDROSE_METRICS_ENABLED:-false} +METRICS_BIND=\${METRICS_BIND:-127.0.0.1} +METRICS_PORT=\${METRICS_PORT:-9464} WINDROSE_DISCORD_WEBHOOK_URL=\${WINDROSE_DISCORD_WEBHOOK_URL:-} WINDROSE_WEBHOOK_URL=\${WINDROSE_WEBHOOK_URL:-} WINDROSE_WEBHOOK_EVENTS=\${WINDROSE_WEBHOOK_EVENTS:-server.online,server.offline,player.join,player.leave,backup.created,backup.restored,config.applied} @@ -136,6 +146,10 @@ assert SERVER_NAME "My Server" || fa assert MAX_PLAYER_COUNT "6" || fail=1 assert P2P_PROXY_ADDRESS "192.168.1.50" || fail=1 assert WINDROSE_PATCH_IDLE_CPU "1" || fail=1 +assert UI_ENABLE_METRICS_ROUTE "false" || fail=1 +assert WINDROSE_METRICS_ENABLED "true" || fail=1 +assert METRICS_BIND "127.0.0.1" || fail=1 +assert METRICS_PORT "9464" || fail=1 assert CUSTOM_OPERATOR_KEY "custom_value" || fail=1 # Also assert unset-in-existing vars fall back to their defaults. @@ -171,17 +185,21 @@ WINDROSE_ENV_FILE="${_tmp}/does-not-exist-$$.env" # no prior env file : "\${UI_PORT:=28080}" : "\${UI_PASSWORD:=}" : "\${UI_ENABLE_ADMIN_WITHOUT_PASSWORD:=false}" +: "\${UI_ENABLE_METRICS_ROUTE:=false}" +: "\${WINDROSE_METRICS_ENABLED:=false}" +: "\${METRICS_BIND:=127.0.0.1}" +: "\${METRICS_PORT:=9464}" # Simulated status echo — the thing that would fail under set -u # without the fix. -echo "bind=\${UI_BIND} port=\${UI_PORT} pwlen=\${#UI_PASSWORD} admin=\${UI_ENABLE_ADMIN_WITHOUT_PASSWORD}" > '${out2}' +echo "bind=\${UI_BIND} port=\${UI_PORT} pwlen=\${#UI_PASSWORD} admin=\${UI_ENABLE_ADMIN_WITHOUT_PASSWORD} metrics=\${WINDROSE_METRICS_ENABLED} mbind=\${METRICS_BIND} mport=\${METRICS_PORT}" > '${out2}' HARNESS if [ ! -f "${out2}" ]; then echo " FAIL fresh install set -u aborted before echo" exit 1 fi -if ! grep -q "bind=127.0.0.1 port=28080 pwlen=0 admin=false" "${out2}"; then +if ! grep -q "bind=127.0.0.1 port=28080 pwlen=0 admin=false metrics=false mbind=127.0.0.1 mport=9464" "${out2}"; then echo " FAIL fresh-install defaults incorrect:" cat "${out2}" exit 1 diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..446f708 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Tests for the Prometheus metrics exporter.""" + +from __future__ import annotations + +import json +import sys +import tempfile +import threading +import urllib.error +import urllib.request +from http.server import ThreadingHTTPServer +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import metrics # noqa: E402 +import server # noqa: E402 + + +WORLD_ID = "ABCDEF0123456789ABCDEF0123456789" + + +def _patch_paths(root: Path) -> tuple[Path, Path]: + windrose_dir = root / "WindowsServer" + r5 = windrose_dir / "R5" + backup_root = root / "backups" + server.WINDROSE_SERVER_DIR = windrose_dir + server.R5_DIR = r5 + server.SAVE_ROOT = r5 / "Saved" / "SaveProfiles" / "Default" / "RocksDB" + server.R5_LOG = r5 / "Saved" / "Logs" / "R5.log" + server.CONFIG_PATH = r5 / "ServerDescription.json" + server.STAGED_CONFIG_PATH = r5 / "ServerDescription.staged.json" + server.BACKUP_ROOT = backup_root + server.GAME_EXE_PATH = r5 / "Binaries" / "Win64" / "WindroseServer-Win64-Shipping.exe" + server.MAINTENANCE_FLAG_FILE = r5 / ".maintenance-mode" + server.CPU_STATE_PATH = root / "cpu.state" + return r5, backup_root + + +def _seed(root: Path) -> None: + r5, backup_root = _patch_paths(root) + world = server.SAVE_ROOT / "0.10.0" / "Worlds" / WORLD_ID + world.mkdir(parents=True) + (world / "WorldDescription.json").write_text(json.dumps({ + "WorldDescription": { + "WorldName": "Metrics World", + "WorldPresetType": "Medium", + } + })) + server.CONFIG_PATH.write_text(json.dumps({ + "ServerDescription_Persistent": { + "ServerName": "metrics-test", + "MaxPlayerCount": 4, + "IsPasswordProtected": False, + "Password": "", + "P2pProxyAddress": "127.0.0.1", + "PersistentServerId": "11111111111111111111111111111111", + "InviteCode": "ABC123", + "WorldIslandId": WORLD_ID, + } + })) + server.R5_LOG.parent.mkdir(parents=True, exist_ok=True) + server.R5_LOG.write_text( + "r5coopapigateway-kr-release.windrose.support\n" + "GameVersion 0.10.0.5.120-073042fb. " + "ShaVersion 073042fb338d004c3e94d18ef0745fb210fa9fdf. " + "ReleaseVersion 0.10.0. DeploymentId 0.10.0.5.120-073042fb.\n" + ) + manifest = server.WINDROSE_SERVER_DIR / "steamapps" / "appmanifest_4129620.acf" + manifest.parent.mkdir(parents=True) + manifest.write_text('"buildid"\t\t"23065343"\n') + server.GAME_EXE_PATH.parent.mkdir(parents=True) + server.GAME_EXE_PATH.write_bytes(b"fake exe") + (r5 / ".mods.json").write_text(json.dumps({ + "schemaVersion": 1, + "mods": [{"id": "testmod", "displayName": "Test Mod", "enabled": True}], + })) + backup = backup_root / "manual-20260504T170728Z" + backup.mkdir(parents=True) + (backup / "Saved").mkdir() + (backup / "Saved" / "x").write_bytes(b"backup") + auto = backup_root / "20260504T180000Z" + auto.mkdir() + (auto / server.AUTO_BACKUP_MARKER_NAME).write_text("") + + +def test_render_metrics_contains_aggregate_state() -> None: + with tempfile.TemporaryDirectory() as tmp: + _seed(Path(tmp)) + text = metrics.render_metrics() + assert "windrose_server_running 0" in text + assert "windrose_players_max 4.0" in text + assert 'windrose_backend_region_info{region="kr"} 1.0' in text + assert 'steam_buildid="23065343"' in text + assert 'game_version="0.10.0.5.120-073042fb"' in text + assert 'windrose_mods_total{state="enabled"} 1.0' in text + assert 'windrose_backups_total{source="manual-pinned"} 1.0' in text + assert 'windrose_backups_total{source="auto"} 1.0' in text + assert "windrose_exporter_scrape_success 1" in text + + +def test_standalone_metrics_http_handler() -> None: + with tempfile.TemporaryDirectory() as tmp: + _seed(Path(tmp)) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), metrics.MetricsHandler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + raw = urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/metrics", timeout=5).read().decode() + assert "windrose_server_running" in raw + finally: + httpd.shutdown() + httpd.server_close() + + +def test_admin_ui_metrics_route_is_opt_in_and_open() -> None: + with tempfile.TemporaryDirectory() as tmp: + _seed(Path(tmp)) + old_enabled = server.UI_ENABLE_METRICS_ROUTE + old_password = server.UI_PASSWORD + try: + server.UI_ENABLE_METRICS_ROUTE = True + server.UI_PASSWORD = "secret" + httpd = ThreadingHTTPServer(("127.0.0.1", 0), server.Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + raw = urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/metrics", timeout=5).read().decode() + assert "windrose_server_running" in raw + finally: + server.UI_ENABLE_METRICS_ROUTE = old_enabled + server.UI_PASSWORD = old_password + httpd.shutdown() + httpd.server_close() + + +def test_admin_ui_metrics_route_disabled_returns_not_found() -> None: + with tempfile.TemporaryDirectory() as tmp: + _seed(Path(tmp)) + old_enabled = server.UI_ENABLE_METRICS_ROUTE + old_password = server.UI_PASSWORD + try: + server.UI_ENABLE_METRICS_ROUTE = False + server.UI_PASSWORD = "secret" + httpd = ThreadingHTTPServer(("127.0.0.1", 0), server.Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + urllib.request.urlopen(f"http://127.0.0.1:{httpd.server_address[1]}/metrics", timeout=5) + raise AssertionError("/metrics unexpectedly succeeded") + except urllib.error.HTTPError as e: + assert e.code == 404 + finally: + server.UI_ENABLE_METRICS_ROUTE = old_enabled + server.UI_PASSWORD = old_password + httpd.shutdown() + httpd.server_close() + + +if __name__ == "__main__": + test_render_metrics_contains_aggregate_state() + test_standalone_metrics_http_handler() + test_admin_ui_metrics_route_is_opt_in_and_open() + test_admin_ui_metrics_route_disabled_returns_not_found() + print("metrics tests passed")