feat: telemetry surface (/status extras, /metrics, /usage)

.env.example

@@ -32,3 +32,8 @@ HEALTH_POLL_INTERVAL_MS=1000
 # How long upstream chat-completions calls may take. Generous default for
 # slow CPU / cold-cache generations on local models.
 REQUEST_TIMEOUT_MS=600000
+
+# How many recent request records jano keeps in memory for `GET /usage`
+# (the "what did my last N requests cost" log). Older records drop FIFO.
+# Purely observability; bump it for a longer benchmark history.
+USAGE_RECORDS_MAX=500

README.md

@@ -150,17 +150,103 @@ Returns the configured models in OpenAI's list format. Useful for clients that p
 
 ### `GET /health` and `GET /status`
 
-Both return the same payload, jano's view of the world:
+Both return the same payload: jano's full view of the world. Because jano sits
+_above_ the backends and owns every request, response, and swap, it can report
+telemetry no single backend (Ollama, `llama-server`, `mlx-lm.server`) can
+produce on its own — queue backlog, swap economics, rolling throughput across
+all callers, and cumulative counters.
 
-```json
+```jsonc
 {
   "ok": true,
   "currentModel": "chat",
+  "currentModelLoadedAt": "2026-06-19T18:02:11.000Z",
+
+  // Queue & concurrency (router-only).
   "queueDepth": 0,
-  "queueByModel": { "chat": 0, "code": 0, "fast": 0 }
+  "queueByModel": { "chat": 0, "code": 0 },
+  "inFlight": 1,
+  "oldestWaitingMs": 0, // age of the head-of-line waiting request
+
+  // Swap economics (router-only — only jano invokes the swap).
+  "lastSwapDurationMs": 21840,
+  "swapsLast15m": 1,
+  "swapsLastHour": 3,
+
+  // Aggregate "lately" throughput across ALL callers (rolling, last N).
+  "recentGenTokS": 47.2,
+  "recentPromptTokS": 312.5,
+  "tokSByModel": { "chat": 51.0, "code": 39.4 },
+
+  // Cumulative counters since start.
+  "uptimeSeconds": 84211,
+  "requestsServedTotal": 128,
+  "requestsByModel": { "chat": 90, "code": 38 },
+  "tokensGeneratedTotal": 51234,
+  "tokensByModel": { "chat": 33001, "code": 18233 },
+  "requestsPerMinute": 4,
+
+  // Error & health signal.
+  "errorsLast15m": 0,
+  "lastError": null,
+  "backendHealth": { "chat": "up", "code": "up" },
 }
 ```
 
+Token counts and tok/s come from the backend's response when it reports them —
+jano normalizes across shapes: OpenAI `usage`, llama.cpp `timings`, and
+Ollama-style nanosecond fields (`eval_count`/`eval_duration`). Generation tok/s
+is otherwise derived from measured time-to-first-byte and completion. When a
+backend reports nothing usable, the relevant fields are simply `null` —
+telemetry degrades gracefully rather than guessing.
+
+### `GET /metrics`
+
+Prometheus text exposition (`v0.0.4`) of the cumulative counters, swap-duration
+histogram, queue gauges, and per-backend health — scrape it straight into
+Grafana, or read it from a client. Sample:
+
+```text
+jano_uptime_seconds 84211
+jano_in_flight 1
+jano_queue_depth{model="chat"} 0
+jano_requests_served_total{model="chat"} 90
+jano_tokens_generated_total{model="chat"} 33001
+jano_swaps_total{from="chat",to="code"} 3
+jano_swap_transition_duration_milliseconds_sum{from="chat",to="code"} 65520
+jano_swap_duration_milliseconds_bucket{le="30000"} 3
+jano_backend_up{model="chat"} 1
+```
+
+### `GET /usage?limit=50`
+
+A ring buffer of the most recent requests (newest first), each with token
+counts and timings — "what did my last N requests cost in compute." The buffer
+size is `USAGE_RECORDS_MAX` (default 500).
+
+```jsonc
+{
+  "count": 2,
+  "records": [
+    {
+      "ts": "2026-06-19T18:05:42.000Z",
+      "model": "chat",
+      "status": 200,
+      "total_ms": 1840,
+      "ttfb_ms": 120,
+      "prompt_tokens": 312,
+      "gen_tokens": 87,
+      "gen_tok_s": 51.0,
+      "prompt_tok_s": 410.2,
+    },
+  ],
+}
+```
+
+> **Out of scope:** GPU temperature, utilization, fan speed, and system
+> RAM/CPU/power are hardware readings outside any router's reach. Jano does
+> not and cannot report them — that's a host-side helper's job.
+
 ## The swap script contract
 
 Jano shells out to `SWAP_COMMAND` to swap models. The contract is intentionally small:
@@ -205,6 +291,7 @@ All via env vars; copy `.env.example` to `.env` to start.
 | `SWAP_WAIT_TIMEOUT_MS`    | no       | `180000`        | Max time jano waits for a backend to become healthy after a swap.          |
 | `HEALTH_POLL_INTERVAL_MS` | no       | `1000`          | How often jano pings `/health` while waiting.                              |
 | `REQUEST_TIMEOUT_MS`      | no       | `600000`        | Per-request upstream timeout. Generous default for cold/large generations. |
+| `USAGE_RECORDS_MAX`       | no       | `500`           | Size of the in-memory recent-request ring served by `GET /usage`.          |
 
 ## When you don't need jano
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jano",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "OpenAI-compatible router that batches LLM requests by model so a flurry of mixed calls costs one swap, not many. Backend-agnostic; works with any number of models.",
   "private": true,
   "type": "module",

src/bodyTap.ts

@@ -0,0 +1,60 @@
+import { Buffer } from 'node:buffer';
+
+/**
+ * Captures just enough of a forwarded response body to extract token/timing
+ * telemetry, without buffering the whole thing or perturbing what we stream
+ * to the client.
+ *
+ * Two modes, chosen by `tailOnly`:
+ *
+ *   - **Streaming (`tailOnly = true`)**: keep only the last `cap` bytes. The
+ *     `usage`/`timings` payload an SSE response carries always lives in the
+ *     final `data:` frame, so a small tail window captures it while bounding
+ *     memory regardless of how long the generation runs.
+ *   - **Non-streaming (`tailOnly = false`)**: a chat-completions JSON body is
+ *     small, so accumulate the whole thing — but stop at `cap` to stay safe
+ *     against a pathological upstream. If we overflow we simply decline to
+ *     parse (telemetry records null token counts rather than risk garbage).
+ *
+ * Feeding chunks is allocation-light: we hold references and only concat once,
+ * lazily, in `text()`.
+ */
+export class BodyTap {
+  private chunks: Buffer[] = [];
+  private bytes = 0;
+  private overflowed = false;
+  private readonly tailOnly: boolean;
+  private readonly cap: number;
+
+  constructor(tailOnly: boolean, cap = 64 * 1024) {
+    this.tailOnly = tailOnly;
+    this.cap = cap;
+  }
+
+  push(chunk: Buffer): void {
+    if (this.tailOnly) {
+      this.chunks.push(chunk);
+      this.bytes += chunk.length;
+      // Drop from the front until we're back under cap, but always keep at
+      // least the most recent chunk so a single oversized frame still parses.
+      while (this.bytes > this.cap && this.chunks.length > 1) {
+        const removed = this.chunks.shift();
+        if (removed) this.bytes -= removed.length;
+      }
+      return;
+    }
+    if (this.overflowed) return;
+    if (this.bytes + chunk.length > this.cap) {
+      this.overflowed = true;
+      return;
+    }
+    this.chunks.push(chunk);
+    this.bytes += chunk.length;
+  }
+
+  /** Decoded captured bytes, or null if a non-streaming body overflowed. */
+  text(): string | null {
+    if (this.overflowed) return null;
+    return Buffer.concat(this.chunks).toString('utf8');
+  }
+}

src/config.ts

@@ -135,6 +135,11 @@ export const env = {
   /** Reset a model's swap-failure count after it has been quiet this long;
    *  lets the dispatcher retry once a broken backend may have recovered. */
   SWAP_FAILURE_RESET_MS: optInt('SWAP_FAILURE_RESET_MS', 5 * 60_000),
+
+  /** Size of the in-memory recent-request ring served by `GET /usage`. Older
+   *  records are dropped FIFO. Purely an observability buffer; bump it if you
+   *  want a longer benchmark history at the cost of a little memory. */
+  USAGE_RECORDS_MAX: optInt('USAGE_RECORDS_MAX', 500),
 };
 
 export const models: ModelDef[] = loadModels(env.MODELS_FILE);

src/dispatcher.ts

@@ -1,8 +1,9 @@
 import type { ModelName } from './types.ts';
 import { env, modelNames } from './config.ts';
 import { FailureBudget } from './failureBudget.ts';
-import { _internal, getQueueByModel, getQueueDepth } from './queue.ts';
+import { _internal, getOldestEnqueuedAt, getQueueByModel, getQueueDepth } from './queue.ts';
 import { detectLoaded, ensureLoaded } from './swap.ts';
+import { telemetry } from './metrics.ts';
 import { log } from './log.ts';
 
 let currentModel: ModelName | null = null;
@@ -15,11 +16,29 @@ let running = false;
 // Reproduces the 2026-05-12 wedge fix.
 const swapFailures = new FailureBudget(env.SWAP_FAILURE_LIMIT, env.SWAP_FAILURE_RESET_MS);
 
+/** A request that never reached a backend (dead client, swap failure) still
+ *  belongs in the usage history with its short-circuit status and no tokens. */
+function recordShortCircuit(model: ModelName, status: number, enqueuedAt: number): void {
+  telemetry.recordRequest({
+    model,
+    status,
+    totalMs: Date.now() - enqueuedAt,
+    ttfbMs: null,
+    promptTokens: null,
+    genTokens: null,
+    genTokS: null,
+    promptTokS: null,
+  });
+}
+
 export function getStatus() {
+  const oldest = getOldestEnqueuedAt();
   return {
     currentModel,
     queueDepth: getQueueDepth(),
     queueByModel: getQueueByModel(modelNames()),
+    oldestWaitingMs: oldest === null ? 0 : Math.max(0, Date.now() - oldest),
+    ...telemetry.snapshot(),
   };
 }
 
@@ -35,6 +54,7 @@ async function loop(): Promise<void> {
 
     if (!task.isAlive()) {
       log.info('dropping disconnected task', { id: task.id, model: task.model });
+      recordShortCircuit(task.model, 499, task.enqueuedAt);
       await task.fail(499, 'client disconnected before dispatch');
       continue;
     }
@@ -47,14 +67,18 @@ async function loop(): Promise<void> {
           model: task.model,
           fails,
         });
+        telemetry.recordError(task.model, `fail-fast: ${fails} consecutive swap failures`);
+        recordShortCircuit(task.model, 503, task.enqueuedAt);
         await task.fail(
           503,
           `backend "${task.model}" is unavailable (${fails} consecutive swap failures; auto-retry in ~${Math.round(env.SWAP_FAILURE_RESET_MS / 1000)}s)`
         );
         continue;
       }
+      const swapStart = Date.now();
       try {
         await ensureLoaded(task.model);
+        telemetry.recordSwap(currentModel, task.model, Date.now() - swapStart);
         currentModel = task.model;
         swapFailures.recordSuccess(task.model);
       } catch (err) {
@@ -65,16 +89,24 @@ async function loop(): Promise<void> {
           fails: newFails,
           err: String(err),
         });
+        telemetry.recordError(task.model, `swap failed: ${String(err)}`);
+        recordShortCircuit(task.model, 503, task.enqueuedAt);
         await task.fail(503, `swap to "${task.model}" failed: ${String(err)}`);
         continue;
       }
     }
 
     try {
-      await task.run();
+      telemetry.incInFlight();
+      try {
+        await task.run();
+      } finally {
+        telemetry.decInFlight();
+      }
       swapFailures.recordSuccess(task.model);
     } catch (err) {
       log.error('task threw', { id: task.id, err: String(err) });
+      telemetry.recordError(task.model, `task threw: ${String(err)}`);
     }
   }
   log.info('dispatcher stopped');
@@ -83,7 +115,9 @@ async function loop(): Promise<void> {
 export async function start(): Promise<void> {
   if (running) return;
   running = true;
+  telemetry.seedModels(modelNames());
   currentModel = await detectLoaded();
+  if (currentModel !== null) telemetry.noteInitialModel(currentModel);
   log.info('dispatcher detected initial model', { currentModel });
   void loop();
 }