feat(observability): capture every miner AI generation, and stop fabricating zero token counts (#10233)

* feat(observability): capture every miner AI generation, and stop fabricating zero token counts

Closes #10200
Closes #10207

* chore(contract): regenerate api-schemas for the linkedIssueMaintainerExempt field

#10160 added linkedIssueMaintainerExempt to the repository-settings surface without
regenerating packages/loopover-contract/src/api-schemas.ts, so contract:api-schemas:check
has been failing on main since. The check is part of the local test:ci aggregate but is
not wired into ci.yml, which is why nothing surfaced it.

Pure regeneration: the one added line is the generator's own output for a field that
already exists everywhere else.

---------

Co-authored-by: JSONbored <aetherealdev@gmail.com>

Author: JSONbored

packages/loopover-engine/src/index.ts

@@ -392,12 +392,21 @@ export {
   createFakeCodingAgentDriverForFactory,
   isConfiguredCodingAgentDriver,
   resolveConfiguredCodingAgentDriverNames,
+  resolveCodingAgentTelemetryModel,
   resolveFirstConfiguredCodingAgentDriverName,
   runCodingAgentAttempt,
   type CodingAgentDriverName,
   type CreateCodingAgentDriverOptions,
   type RunCodingAgentAttemptOptions,
 } from "./miner/driver-factory.js";
+export {
+  emitMinerAiGeneration,
+  hasMinerAiGenerationSink,
+  setMinerAiGenerationSink,
+  withCodingAgentGenerationCapture,
+  type MinerAiGenerationRecord,
+  type MinerAiGenerationSink,
+} from "./miner/ai-generation-sink.js";
 export * from "./miner/attempt-metering.js";
 export {
   buildRepoMap,
@@ -415,6 +424,7 @@ export {
 } from "./miner/repo-map.js";
 export {
   createAgentSdkCodingAgentDriver,
+  readAgentSdkResultUsage,
   type AgentSdkHooks,
   type AgentSdkQueryFn,
   type AgentSdkQueryOptions,
@@ -423,6 +433,7 @@ export {
 export {
   buildChatPrompt,
   CHAT_GROUNDING_MCP_SERVER_NAME,
+  CHAT_GROUNDING_PROVIDER,
   CHAT_GROUNDING_TOOL_NAMES,
   CHAT_REDACTED_TEXT,
   CHAT_SYSTEM_PROMPT,

packages/loopover-engine/src/miner/agent-sdk-driver.ts

@@ -111,6 +111,25 @@ function tokensFromResultMessage(resultMessage: Record<string, unknown> | null):
   };
 }
 
+/** The billing facts an SDK `result` frame carries: the token split above plus the session's real dollar cost.
+ *  `SDKResultSuccess`/`SDKResultError` both declare `total_cost_usd: number` unconditionally — present whenever
+ *  a result message arrived at all, success or not (the session was billed either way), absent only when the
+ *  stream produced no result frame. `finiteNonNegativeNumber` (not a bare `typeof`) because a malformed value
+ *  from an untyped source must degrade to undefined rather than propagate (#5827).
+ *
+ *  Exported (#10200) so `chat-grounding.ts` reads the SAME frame the same way. It drives its own `query()`
+ *  session and needs the identical usage/cost facts; a private copy there would be a third instance of these
+ *  defensive reads, which is the duplicated-block class #10170 catalogues. */
+export function readAgentSdkResultUsage(resultMessage: Record<string, unknown> | null): {
+  tokens: CodingAgentTokenUsage;
+  costUsd: number | undefined;
+} {
+  return {
+    tokens: tokensFromResultMessage(resultMessage),
+    costUsd: finiteNonNegativeNumber(resultMessage?.total_cost_usd),
+  };
+}
+
 async function listWorktreeChangedFiles(cwd: string): Promise<string[]> {
   const [tracked, untracked] = await Promise.all([
     execFileAsync("git", ["-C", cwd, "diff", "--name-only", "HEAD", "--"]),
@@ -204,11 +223,7 @@ export function createAgentSdkCodingAgentDriver(
       // negative) must degrade to undefined here, or it reaches accumulateAttemptUsage unguarded and throws a
       // RangeError that rejects runIterateLoopCore before any decision is logged (#5827).
       const turnsUsed = finiteNonNegativeNumber(resultMessage?.num_turns);
-      // Real dollar cost: the SDK's own SDKResultSuccess/SDKResultError message types both declare
-      // `total_cost_usd: number` unconditionally -- present whenever a result message arrived at all, success
-      // or not (the session was billed either way), absent only when the stream produced no result message.
-      const costUsd = finiteNonNegativeNumber(resultMessage?.total_cost_usd);
-      const tokenUsage = tokensFromResultMessage(resultMessage);
+      const { tokens: tokenUsage, costUsd } = readAgentSdkResultUsage(resultMessage);
       const resultText =
         typeof resultMessage?.result === "string" ? redactSecrets(resultMessage.result) : "";
       const transcript = redactSecrets(

packages/loopover-engine/src/miner/ai-generation-sink.ts

@@ -0,0 +1,114 @@
+// Host-registered `$ai_generation` sink for the miner's AMS surfaces (#10200, epic #8286 Phase 3).
+//
+// Every real model call this package drives -- a coding-agent driver attempt (driver-factory.ts) and a
+// chat-grounding session (chat-grounding.ts) -- is spend that belongs in PostHog. The engine cannot capture it
+// itself: it is the portable package, and `posthog-node` (or any other vendor client) is exactly the dependency
+// it must not carry -- the same boundary cli-subprocess-driver.ts's and coding-agent-construction.ts's own
+// headers already document.
+//
+// The previous shape solved that by attaching the capture at a HOST construction site instead
+// (`withCodingAgentAiGenerationCapture`, applied inside `constructProductionCodingAgentDriver`), which made
+// opting out silent: `runCodingAgentAttempt` builds its driver through `createCodingAgentDriver` DIRECTLY and
+// never passes through that site, so every attempt it ran was uncaptured and nothing anywhere said so. A
+// wrapper attached at one of N construction sites is precisely the "two or more places that must agree, with
+// nothing enforcing it" shape #10170 catalogues, and #10127's fix for the same class is the precedent: make the
+// bypass unrepresentable rather than documenting it.
+//
+// So the sink is registered ONCE by the host process and consumed inside the engine at the single chokepoint
+// every real driver is constructed through. A future third construction site cannot silently opt out, because
+// there is nothing left for it to forget to attach.
+
+import type { CodingAgentDriver } from "./coding-agent-driver.js";
+
+/**
+ * One completed model call, in the host-neutral shape a `$ai_generation` capture needs. Metadata only: model and
+ * provider ids, timing, token/cost accounting, and the raw caught value on the error path -- there is
+ * deliberately no field for the prompt, the transcript, or any tool output, matching the ORB side's identical
+ * policy (`PostHogAiGenerationEvent`, src/selfhost/posthog.ts).
+ *
+ * Every token/cost field is OPTIONAL and stays absent when the provider reported nothing (#10207): a fabricated
+ * 0 is indistinguishable from a real 0 in an aggregate, so the sink must be able to tell "zero" from "unknown".
+ */
+export type MinerAiGenerationRecord = {
+  provider: string;
+  model: string;
+  latencyMs: number;
+  isError: boolean;
+  totalTokens?: number | undefined;
+  inputTokens?: number | undefined;
+  outputTokens?: number | undefined;
+  totalCostUsd?: number | undefined;
+  error?: unknown;
+};
+
+/** The host's capture function. Synchronous and fire-and-forget -- a sink that needs to do I/O queues it. */
+export type MinerAiGenerationSink = (record: MinerAiGenerationRecord) => void;
+
+let sink: MinerAiGenerationSink | undefined;
+
+/**
+ * Register (or, with `undefined`, clear) the process-wide sink. Called once by the host during startup --
+ * `initMinerPostHog` (packages/loopover-miner/lib/posthog.ts) registers its own capture here when the operator
+ * has opted in, and registers nothing when they have not, so the no-phone-home default (#6011) is preserved by
+ * construction: with no sink registered, {@link emitMinerAiGeneration} is a no-op.
+ *
+ * Module-level rather than threaded through every call: the same shape posthog.ts's own `client`/`active` module
+ * state already uses on the host side, and for the same reason -- a process has exactly one telemetry sink, and
+ * an optional parameter threaded through N call sites is the opt-out this module exists to remove.
+ */
+export function setMinerAiGenerationSink(next: MinerAiGenerationSink | undefined): void {
+  sink = next;
+}
+
+/** True when a host sink is registered. Exported for the host's own wiring assertions, not for gating a call. */
+export function hasMinerAiGenerationSink(): boolean {
+  return sink !== undefined;
+}
+
+/** Report one completed model call. No-op when no sink is registered, and never throws -- telemetry must never
+ *  crash the AI call it is instrumenting, the same contract every capture function in the miner's posthog.ts
+ *  already holds on its own side. */
+export function emitMinerAiGeneration(record: MinerAiGenerationRecord): void {
+  if (!sink) return;
+  try {
+    sink(record);
+  } catch {
+    /* A sink that throws is a telemetry bug, never the caller's problem. */
+  }
+}
+
+/**
+ * Wrap a real `CodingAgentDriver` so every attempt it runs reports a generation. Moved here from the miner's own
+ * construction site (#10200) so the single engine-side factory can apply it to every driver it builds.
+ *
+ * `CodingAgentDriverResult` carries the blended `tokensUsed` plus the input/output split when the provider
+ * reported one (#10198); all of it is forwarded verbatim, and a driver that knows no split simply leaves those
+ * fields absent rather than having one fabricated. A driver failure is reported via `result.ok === false` (the
+ * real, observed contract every shipped driver follows -- none of them throw for an ordinary task failure), with
+ * a genuine thrown exception handled defensively on top.
+ */
+export function withCodingAgentGenerationCapture(provider: string, model: string, driver: CodingAgentDriver): CodingAgentDriver {
+  return {
+    async run(task) {
+      const startedAtMs = Date.now();
+      try {
+        const result = await driver.run(task);
+        emitMinerAiGeneration({
+          provider,
+          model,
+          latencyMs: Date.now() - startedAtMs,
+          isError: !result.ok,
+          totalTokens: result.tokensUsed,
+          inputTokens: result.inputTokens,
+          outputTokens: result.outputTokens,
+          totalCostUsd: result.costUsd,
+          error: result.ok ? undefined : result.error,
+        });
+        return result;
+      } catch (error) {
+        emitMinerAiGeneration({ provider, model, latencyMs: Date.now() - startedAtMs, isError: true, error });
+        throw error;
+      }
+    },
+  };
+}

packages/loopover-engine/src/miner/chat-grounding.ts

@@ -18,7 +18,9 @@
 // The endpoint is stateless: the caller supplies the full message history per request (no conversation store).
 
 import { PUBLIC_FIELD_BLOCKLIST } from "../track-record-summary.js";
-import { resolveFirstConfiguredCodingAgentDriverName } from "./driver-factory.js";
+import { readAgentSdkResultUsage } from "./agent-sdk-driver.js";
+import { emitMinerAiGeneration } from "./ai-generation-sink.js";
+import { resolveCodingAgentTelemetryModel, resolveFirstConfiguredCodingAgentDriverName } from "./driver-factory.js";
 
 /**
  * The exact read-only tools this endpoint may call — one `server.registerTool(...)` call each in
@@ -42,6 +44,11 @@ export const CHAT_GROUNDING_TOOL_NAMES = Object.freeze([
 /** The MCP server name the session registers the miner tools under. */
 export const CHAT_GROUNDING_MCP_SERVER_NAME = "loopover-miner";
 
+/** The only provider this endpoint can run on (boundary 1 above) — and therefore the provider id its
+ *  `$ai_generation` telemetry reports (#10200). One definition, consumed by both the fail-closed check and the
+ *  capture, so the two can never disagree about what is actually running. */
+export const CHAT_GROUNDING_PROVIDER = "agent-sdk";
+
 /** Ceiling on a single conversational session's tool-calling turns. */
 const CHAT_MAX_TURNS = 12;
 
@@ -201,7 +208,7 @@ export function resolveChatProviderError(
         "No coding-agent provider is configured. Chat requires the agent-sdk provider — set MINER_CODING_AGENT_PROVIDER=agent-sdk.",
     };
   }
-  if (provider !== "agent-sdk") {
+  if (provider !== CHAT_GROUNDING_PROVIDER) {
     return {
       code: "chat_requires_agent_sdk_provider",
       message: `Chat requires the agent-sdk provider; the configured provider is ${provider}, which is a single-turn, buffered coding driver.`,
@@ -254,9 +261,35 @@ function* foldToolResultMessage(message: Record<string, unknown>): Generator<Cha
   }
 }
 
+/**
+ * Why a completed session was not a successful generation, or `undefined` when it was. A stream that ends
+ * without a `result` frame never reported completion, and one whose result frame is itself an error reported
+ * failure — agent-sdk-driver.ts classifies exactly these two the same way (`agent_sdk_no_result` /
+ * `agent_sdk_<subtype>`), and the telemetry must not call either of them a success.
+ */
+function chatResultFailureReason(resultMessage: Record<string, unknown> | null): string | undefined {
+  if (!resultMessage) return "chat_grounding_no_result";
+  if (resultMessage.is_error === true) return "chat_grounding_errored";
+  if (resultMessage.subtype !== "success") {
+    return `chat_grounding_${typeof resultMessage.subtype === "string" ? resultMessage.subtype : "unknown"}`;
+  }
+  return undefined;
+}
+
 /**
  * Drives one grounded conversational turn, yielding wire events. Never throws: an SDK failure becomes an `error`
  * event, and `done` always terminates the stream — including on the fail-closed provider paths.
+ *
+ * #10200: this drives a real `query()` session with a real turn budget, so it is real spend and reports an
+ * `$ai_generation` through the host sink (ai-generation-sink.ts) on both the completed and the thrown path.
+ * That also means folding the SDK's `result` frame, which this module previously discarded — it read only the
+ * `assistant`/`user` messages it turns into wire events, so the session's own usage and cost were on the wire
+ * and thrown away.
+ *
+ * The fail-closed provider path above deliberately emits NOTHING: no model was ever reached, so an
+ * `$ai_generation` there would fabricate a generation that did not happen. That case is a request which produced
+ * no generation — the shape the ORB side gives its own separate `selfhost_ai_degraded` event (#10186), which the
+ * miner has no counterpart for yet.
  */
 export async function* runChatGrounding(
   messages: ChatMessage[],
@@ -272,6 +305,9 @@ export async function* runChatGrounding(
 
   const query = resolveChatQuery(options);
   const mcpServer = options.mcpServer ?? DEFAULT_MCP_SERVER;
+  const model = resolveCodingAgentTelemetryModel(CHAT_GROUNDING_PROVIDER, env);
+  const startedAtMs = Date.now();
+  let resultMessage: Record<string, unknown> | null = null;
   try {
     const stream = query({
       prompt: buildChatPrompt(messages),
@@ -289,9 +325,33 @@ export async function* runChatGrounding(
       }
       if (message.type === "user") {
         yield* foldToolResultMessage(message);
+        continue;
       }
+      // Kept, not re-emitted: the result frame carries usage/cost, never conversational content, so it feeds
+      // the capture below and never becomes a wire event.
+      if (message.type === "result") resultMessage = message;
     }
+    const { tokens, costUsd } = readAgentSdkResultUsage(resultMessage);
+    const failure = chatResultFailureReason(resultMessage);
+    emitMinerAiGeneration({
+      provider: CHAT_GROUNDING_PROVIDER,
+      model,
+      latencyMs: Date.now() - startedAtMs,
+      isError: failure !== undefined,
+      totalTokens: tokens.tokensUsed,
+      inputTokens: tokens.inputTokens,
+      outputTokens: tokens.outputTokens,
+      totalCostUsd: costUsd,
+      ...(failure === undefined ? {} : { error: new Error(failure) }),
+    });
   } catch (error) {
+    emitMinerAiGeneration({
+      provider: CHAT_GROUNDING_PROVIDER,
+      model,
+      latencyMs: Date.now() - startedAtMs,
+      isError: true,
+      error,
+    });
     yield {
       type: "error",
       code: "chat_grounding_failed",

packages/loopover-engine/src/miner/driver-factory.ts

@@ -29,6 +29,7 @@ import {
   type AgentSdkHooks,
   type AgentSdkQueryFn,
 } from "./agent-sdk-driver.js";
+import { withCodingAgentGenerationCapture } from "./ai-generation-sink.js";
 
 /** Provider names the factory resolves: the two concrete drivers from #4266/#4267 (`claude-cli`/`codex-cli`
  *  spawn the respective CLI; `agent-sdk` runs in-process via the Agent SDK) plus the `noop` stub. All are
@@ -200,17 +201,27 @@ function createCliProvider(
   });
 }
 
-/** Resolve a concrete driver for `providerName`. Throws on unknown/unconfigured providers (fail-closed). */
-export function createCodingAgentDriver(options: CreateCodingAgentDriverOptions): CodingAgentDriver {
-  if (options.driver) return options.driver;
-  const name = options.providerName.trim().toLowerCase();
-  const env = options.env ?? {};
-  if (!isConfiguredCodingAgentDriver(name, env)) {
-    throw new Error(`unconfigured_coding_agent_driver:${name}`);
-  }
+/** The model id telemetry should report for `name`: the provider's configured model env var when it declares
+ *  one, else the provider name itself — `agent-sdk` declares none (its session uses the account/CLI default),
+ *  mirroring ai.ts's own "unconfigured → a sensible default" convention.
+ *
+ *  Moved here from the miner's construction site (#10200) so every construction path names the model
+ *  identically instead of each one re-deriving it. Uses `firstConfiguredEnvValue`, the same reader
+ *  `createCliProvider` already applies to the same env var, so a whitespace-only value resolves to the provider
+ *  name rather than being passed through as a blank model id. */
+export function resolveCodingAgentTelemetryModel(name: string, env: Record<string, string | undefined>): string {
+  const modelEnvKey = CODING_AGENT_DRIVER_CONFIG_ENV[name as CodingAgentDriverName]?.model;
+  return (modelEnvKey ? firstConfiguredEnvValue(env[modelEnvKey]) : undefined) ?? name;
+}
+
+/** The provider switch itself. Split out of {@link createCodingAgentDriver} so the capture wrapper below has
+ *  exactly one expression to wrap — a provider arm cannot return around it. */
+function createProviderDriver(
+  name: string,
+  options: CreateCodingAgentDriverOptions,
+  env: Record<string, string | undefined>,
+): CodingAgentDriver {
   switch (name) {
-    case "noop":
-      return createNoopCodingAgentDriver();
     case "claude-cli":
       return createCliProvider("claude", "MINER_CODING_AGENT_CLAUDE_MODEL", options, env);
     case "codex-cli":
@@ -228,6 +239,34 @@ export function createCodingAgentDriver(options: CreateCodingAgentDriverOptions)
   }
 }
 
+/** Resolve a concrete driver for `providerName`. Throws on unknown/unconfigured providers (fail-closed).
+ *
+ *  #10200: this is the ONE place a real coding-agent driver is constructed, so it is where `$ai_generation`
+ *  capture is attached — both `constructProductionCodingAgentDriver` (the miner CLI) and
+ *  `resolveDriverForAttempt` (`runCodingAgentAttempt`, below) reach a provider through here, and the second one
+ *  previously produced uncaptured attempts because the wrapper lived at only the first. The capture itself is
+ *  host-supplied (see ai-generation-sink.ts); with no host sink registered it is a no-op. */
+export function createCodingAgentDriver(options: CreateCodingAgentDriverOptions): CodingAgentDriver {
+  // Test seam: an injected driver is returned verbatim and uncaptured. It never reaches a model, so wrapping it
+  // would report generations that did not happen — the same reasoning the `noop` arm below rests on.
+  if (options.driver) return options.driver;
+  const name = options.providerName.trim().toLowerCase();
+  const env = options.env ?? {};
+  if (!isConfiguredCodingAgentDriver(name, env)) {
+    throw new Error(`unconfigured_coding_agent_driver:${name}`);
+  }
+  // `noop` is a stub that makes no model call, so it has no generation to report. Capturing it would fabricate
+  // an $ai_generation for an attempt that never reached a provider — #10207's never-fabricate rule applied to
+  // the event itself rather than to its token fields. (`resolveDriverForAttempt` reaches the same conclusion
+  // independently for dry-run/paused attempts, which bypass this factory entirely.)
+  if (name === "noop") return createNoopCodingAgentDriver();
+  return withCodingAgentGenerationCapture(
+    name,
+    resolveCodingAgentTelemetryModel(name, env),
+    createProviderDriver(name, options, env),
+  );
+}
+
 export type RunCodingAgentAttemptOptions = {
   providerName: string;
   env?: Record<string, string | undefined> | undefined;