docs(ui): document the backtest/calibration system as a fumadocs page (#8114) (#8153)

#8138 (threshold backtest, shipped ORB-native in #8142) and #8139 (logic
backtest CI check, shipped in #8147) are exactly the #3047 failure shape --
new CLI flags, a new CI check with its own trigger/comment behavior, a
gate-adjacent pending policy decision (#8105) -- with no docs. Adds
content/docs/backtest-calibration.mdx (corpus model, the two backtest
mechanisms and why they run where they run, how to read the Pareto-floor
comparison, secret_leak's permanent exclusion, the track-record/corpus-export
CLIs, and the advisory-only guarantee with #8105 noted as pending) plus the
thin route, nav/command-palette entries, and the regenerated route tree.
The two pinned docs-route-count tests move 48 -> 49.

Author: JSONbored

apps/loopover-ui/content/docs/backtest-calibration.mdx

@@ -0,0 +1,79 @@
+---
+title: Backtest & calibration
+description: How LoopOver measures whether its own review rules are right — and backtests threshold and logic changes against real history before they ship.
+---
+
+## What this is
+
+Every time a configured gate blocker fires, LoopOver records it. Every time a human later
+overrides that call (or confirms it), LoopOver records that too. Paired together, the two
+histories become a labeled corpus: _this rule fired against this PR, and a person later said it
+was right ("confirmed") or wrong ("reversed")_.
+
+That corpus makes a rule change **testable before it merges**. A PR that tunes a confidence
+threshold or rewrites detection logic gets scored against the real recorded history — the same
+targets, the same raw inputs — instead of being eyeballed.
+
+<Callout variant="note">
+  Everything on this page is advisory-only today. A backtest verdict — even a regression — never
+  blocks a merge. Whether a REGRESSED verdict should ever gate merges is a tracked, deliberately
+  separate decision that waits for real production track-record data.
+</Callout>
+
+## The corpus
+
+- **Fired events** — recorded when a configured blocker actually carries gate authority, with
+  bounded raw context: the issue text, PR title/body, and diff the rule evaluated, and (for the
+  AI-judgment rule `linked_issue_scope_mismatch`) the model's own raw response text.
+- **Override events** — recorded when a human reverses an automated call, or confirms one (for
+  example, an owner closing a PR that was held for low AI confidence confirms the hold).
+- **`secret_leak` is permanently excluded from raw-context capture.** Storing the diff that
+  triggered a leaked-credential finding would store the credential itself in the audit trail.
+  This rule can therefore never be logic-backtested — by design, not omission.
+
+## Two backtests, two mechanisms
+
+**Threshold changes** run inside LoopOver's own review pass. When a PR's diff changes a known
+confidence-threshold constant, the old and new values are replayed as classifiers over the
+rule's recorded history, and the comparison is rendered directly into the unified review
+comment. No code from the PR is ever executed — a threshold is just a number.
+
+**Logic/regex changes** run in a dedicated CI workflow instead, because honestly verifying a
+rewritten detection function requires _executing the PR's own code_ against history — something
+the production review service must never do. The workflow checks out both the PR's head and its
+base, replays each recorded case's captured raw context through both versions of the detection
+function, and posts its own clearly-labeled **"Logic backtest"** PR comment, separate from the
+unified review comment. It triggers only for PRs touching the watched detection-logic paths,
+skips drafts and forks (fork runs get no secrets), and fails open: an infrastructure problem
+produces a skip notice, never a red check.
+
+## Reading the comparison
+
+Both backtests score the same way:
+
+- **"Reversed" is the positive class.** A classifier predicting "reversed" is saying the rule's
+  original firing was wrong. Precision and recall are computed against the human labels, and
+  stay `N/A` when there is not enough decided data — unknown is never coerced to zero.
+- **The Pareto floor decides the verdict.** A candidate that regresses on _any_ axis is
+  **REGRESSED**, even if the other axis improved. Trading precision for recall is a regression,
+  not a net win. Otherwise the verdict is _improved_ (some axis moved up) or _unchanged_.
+- Cases recorded before raw-context capture existed are skipped and counted in the comment —
+  replaying them against empty inputs would bias both sides, so they are never scored.
+
+## The track record
+
+Every backtest run persists its comparison as structured audit data. The accumulated record —
+how often runs regress, per rule — is the evidence base for the pending merge-gating decision,
+readable at any time with the maintainer CLI:
+
+```bash
+npx tsx scripts/backtest-track-record.ts --db loopover --remote
+```
+
+The corpus itself can be exported as a versioned, checksummed snapshot:
+
+```bash
+npx tsx scripts/backtest-corpus-export.ts --rule-id <ruleId> --output corpus.json --remote
+```
+
+Both CLIs are strictly read-only against the database.

apps/loopover-ui/src/components/site/command-palette.tsx

@@ -61,6 +61,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [
   { label: "Scoreability", to: "/docs/scoreability", group: "Docs" },
   { label: "Upstream drift", to: "/docs/upstream-drift", group: "Docs" },
   { label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" },
+  { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" },
   { label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" },
   { label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" },
   { label: "API reference", to: "/api", group: "Reference" },

apps/loopover-ui/src/components/site/docs-nav.tsx

@@ -99,6 +99,7 @@ export const docsNav: DocsGroup[] = [
       { to: "/docs/branch-analysis", label: "Branch analysis" },
       { to: "/docs/scoreability", label: "Scoreability" },
       { to: "/docs/upstream-drift", label: "Upstream drift" },
+      { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
     ],
   },
   {

apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts

@@ -29,7 +29,7 @@ describe("docs.*.tsx routes never import docs-source(.server) directly", () => {
   const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name));
 
   it("found the expected set of in-scope docs route files", () => {
-    expect(inScope.length).toBe(48);
+    expect(inScope.length).toBe(49);
   });
 
   it.each(inScope)(

apps/loopover-ui/src/routeTree.gen.ts

@@ -63,6 +63,7 @@ import { Route as DocsFederatedFleetIntelligenceRouteImport } from './routes/doc
 import { Route as DocsCapacityRouteImport } from './routes/docs.capacity'
 import { Route as DocsBranchAnalysisRouteImport } from './routes/docs.branch-analysis'
 import { Route as DocsBetaOnboardingRouteImport } from './routes/docs.beta-onboarding'
+import { Route as DocsBacktestCalibrationRouteImport } from './routes/docs.backtest-calibration'
 import { Route as DocsAmsUnattendedSchedulingRouteImport } from './routes/docs.ams-unattended-scheduling'
 import { Route as DocsAmsSizingRouteImport } from './routes/docs.ams-sizing'
 import { Route as DocsAmsOperationsRunbookRouteImport } from './routes/docs.ams-operations-runbook'
@@ -376,6 +377,11 @@ const DocsBetaOnboardingRoute = DocsBetaOnboardingRouteImport.update({
   path: '/beta-onboarding',
   getParentRoute: () => DocsRoute,
 } as any)
+const DocsBacktestCalibrationRoute = DocsBacktestCalibrationRouteImport.update({
+  id: '/backtest-calibration',
+  path: '/backtest-calibration',
+  getParentRoute: () => DocsRoute,
+} as any)
 const DocsAmsUnattendedSchedulingRoute =
   DocsAmsUnattendedSchedulingRouteImport.update({
     id: '/ams-unattended-scheduling',
@@ -554,6 +560,7 @@ export interface FileRoutesByFullPath {
   '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
   '/docs/ams-sizing': typeof DocsAmsSizingRoute
   '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+  '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
   '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
   '/docs/branch-analysis': typeof DocsBranchAnalysisRoute
   '/docs/capacity': typeof DocsCapacityRoute
@@ -633,6 +640,7 @@ export interface FileRoutesByTo {
   '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
   '/docs/ams-sizing': typeof DocsAmsSizingRoute
   '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+  '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
   '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
   '/docs/branch-analysis': typeof DocsBranchAnalysisRoute
   '/docs/capacity': typeof DocsCapacityRoute
@@ -717,6 +725,7 @@ export interface FileRoutesById {
   '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
   '/docs/ams-sizing': typeof DocsAmsSizingRoute
   '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+  '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
   '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
   '/docs/branch-analysis': typeof DocsBranchAnalysisRoute
   '/docs/capacity': typeof DocsCapacityRoute
@@ -802,6 +811,7 @@ export interface FileRouteTypes {
     | '/docs/ams-operations-runbook'
     | '/docs/ams-sizing'
     | '/docs/ams-unattended-scheduling'
+    | '/docs/backtest-calibration'
     | '/docs/beta-onboarding'
     | '/docs/branch-analysis'
     | '/docs/capacity'
@@ -881,6 +891,7 @@ export interface FileRouteTypes {
     | '/docs/ams-operations-runbook'
     | '/docs/ams-sizing'
     | '/docs/ams-unattended-scheduling'
+    | '/docs/backtest-calibration'
     | '/docs/beta-onboarding'
     | '/docs/branch-analysis'
     | '/docs/capacity'
@@ -964,6 +975,7 @@ export interface FileRouteTypes {
     | '/docs/ams-operations-runbook'
     | '/docs/ams-sizing'
     | '/docs/ams-unattended-scheduling'
+    | '/docs/backtest-calibration'
     | '/docs/beta-onboarding'
     | '/docs/branch-analysis'
     | '/docs/capacity'
@@ -1405,6 +1417,13 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof DocsBetaOnboardingRouteImport
       parentRoute: typeof DocsRoute
     }
+    '/docs/backtest-calibration': {
+      id: '/docs/backtest-calibration'
+      path: '/backtest-calibration'
+      fullPath: '/docs/backtest-calibration'
+      preLoaderRoute: typeof DocsBacktestCalibrationRouteImport
+      parentRoute: typeof DocsRoute
+    }
     '/docs/ams-unattended-scheduling': {
       id: '/docs/ams-unattended-scheduling'
       path: '/ams-unattended-scheduling'
@@ -1658,6 +1677,7 @@ interface DocsRouteChildren {
   DocsAmsOperationsRunbookRoute: typeof DocsAmsOperationsRunbookRoute
   DocsAmsSizingRoute: typeof DocsAmsSizingRoute
   DocsAmsUnattendedSchedulingRoute: typeof DocsAmsUnattendedSchedulingRoute
+  DocsBacktestCalibrationRoute: typeof DocsBacktestCalibrationRoute
   DocsBetaOnboardingRoute: typeof DocsBetaOnboardingRoute
   DocsBranchAnalysisRoute: typeof DocsBranchAnalysisRoute
   DocsCapacityRoute: typeof DocsCapacityRoute
@@ -1711,6 +1731,7 @@ const DocsRouteChildren: DocsRouteChildren = {
   DocsAmsOperationsRunbookRoute: DocsAmsOperationsRunbookRoute,
   DocsAmsSizingRoute: DocsAmsSizingRoute,
   DocsAmsUnattendedSchedulingRoute: DocsAmsUnattendedSchedulingRoute,
+  DocsBacktestCalibrationRoute: DocsBacktestCalibrationRoute,
   DocsBetaOnboardingRoute: DocsBetaOnboardingRoute,
   DocsBranchAnalysisRoute: DocsBranchAnalysisRoute,
   DocsCapacityRoute: DocsCapacityRoute,

apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx

@@ -45,7 +45,7 @@ describe("docs route Suspense fallback (#6982)", () => {
       "docs.tsx",
     ]);
     const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name));
-    expect(inScope.length).toBe(48);
+    expect(inScope.length).toBe(49);
 
     for (const file of inScope) {
       const source = readFileSync(join(routesDir, file), "utf8");

apps/loopover-ui/src/routes/docs.backtest-calibration.tsx

@@ -0,0 +1,50 @@
+import { createFileRoute, notFound } from "@tanstack/react-router";
+import { Suspense } from "react";
+
+import { DocsPage } from "@/components/site/docs-page";
+import { LoadingState } from "@/components/site/state-views";
+import { docsClientLoader } from "@/lib/docs-client-loader";
+import { getDocPage } from "@/lib/docs-source.functions";
+
+// Rendered from content/docs/backtest-calibration.mdx via fumadocs-mdx's browser entry
+// (docsClientLoader), through the existing DocsPage/Callout/CodeBlock primitives -- not fumadocs-ui's
+// bundled components. See docs-source.server.ts's comment for why the loader below resolves only a
+// plain, serializable path string.
+export const Route = createFileRoute("/docs/backtest-calibration")({
+  loader: async () => {
+    const page = await getDocPage({ data: { slugs: ["backtest-calibration"] } });
+    if (!page) throw notFound();
+    return page;
+  },
+  head: () => ({
+    meta: [
+      { title: "Backtest & calibration — LoopOver docs" },
+      {
+        name: "description",
+        content:
+          "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.",
+      },
+      { property: "og:title", content: "Backtest & calibration — LoopOver docs" },
+      {
+        property: "og:description",
+        content:
+          "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.",
+      },
+      { property: "og:url", content: "/docs/backtest-calibration" },
+    ],
+    links: [{ rel: "canonical", href: "/docs/backtest-calibration" }],
+  }),
+  component: BacktestCalibrationDoc,
+});
+
+function BacktestCalibrationDoc() {
+  const { path, title, description } = Route.useLoaderData();
+  const Content = docsClientLoader.getComponent(path);
+  return (
+    <DocsPage eyebrow="Core concepts" title={title} description={description}>
+      <Suspense fallback={<LoadingState />}>
+        <Content />
+      </Suspense>
+    </DocsPage>
+  );
+}

apps/loopover-ui/src/routes/docs.index.tsx

@@ -94,6 +94,7 @@ const AUDIENCES: Audience[] = [
       { to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" },
       { to: "/docs/upstream-drift", label: "Upstream drift" },
       { to: "/docs/ai-summaries", label: "AI summaries policy" },
+      { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
     ],
   },
   {