feat: server-side citation verification against source text

backend/src/lib/chat/streaming.ts

@@ -25,6 +25,7 @@ import {
   type AskInputsEvent,
   type EditAnnotation,
   devLog,
+  resolveDocLabel,
 } from "./types";
 import { TOOLS, WORKFLOW_TOOLS } from "./tools/toolSchemas";
 import {
@@ -38,9 +39,11 @@ import {
   type CourtlistenerTurnState,
 } from "./tools/toolDispatcher";
 import {
+  readDocumentContent,
   type TurnEditState,
   type TurnReadState,
 } from "./tools/documentOps";
+import { verifyDocumentCitations } from "./verifyCitations";
 
 
 export type AssistantEvent =
@@ -527,15 +530,39 @@ export async function runLLMStream(params: {
   // Parse and emit citations from <CITATIONS> block
   const { citations: parsedCitations, diagnostics: citationDiagnostics } =
     parseCitationsWithDiagnostics(fullText);
-  const citations = buildCitations
-    ? buildCitations(fullText)
-    : parsedCitations.map((c) =>
-        createCitation(
-          c,
-          docIndex,
-          courtlistenerTurnState.casesByClusterId,
-        ),
-      );
+  let citations: unknown[];
+  if (buildCitations) {
+    // Custom builders (tabular) bypass verification; annotations carry no
+    // verification_status and the UI treats a missing status as untrusted.
+    citations = buildCitations(fullText);
+  } else {
+    const rawCitations = parsedCitations.map((c) =>
+      createCitation(
+        c,
+        docIndex,
+        courtlistenerTurnState.casesByClusterId,
+      ),
+    );
+    // Server-side document-quote verification. Fetch each document's extracted
+    // source text at most once per turn (memoized by doc_id), reading only the
+    // bytes already in storage with emitEvents:false so no new events fire and
+    // the air-gap guarantee holds. Case citations pass through untouched.
+    const sourceTextByDocId = new Map<string, Promise<string>>();
+    const getSourceText = (docId: string): Promise<string> => {
+      let pending = sourceTextByDocId.get(docId);
+      if (!pending) {
+        const label = resolveDocLabel(docId, docStore, docIndex);
+        pending = label
+          ? readDocumentContent(label, docStore, () => {}, docIndex, db, {
+              emitEvents: false,
+            })
+          : Promise.resolve("");
+        sourceTextByDocId.set(docId, pending);
+      }
+      return pending;
+    };
+    citations = await verifyDocumentCitations(rawCitations, getSourceText);
+  }
   devLog("[chat/stream] final citations", {
     hasCitationsBlock: citationDiagnostics.hasBlock,
     citationsBlockLength: citationDiagnostics.rawLength,

backend/src/lib/chat/tools/documentOps.ts

@@ -1578,14 +1578,30 @@ export async function readDocumentContent(
   }
 }
 
+/** A character is "punctuation" for tolerant matching if it is not a letter,
+ *  number, or whitespace. Dropped entirely (not replaced with a space) so
+ *  "U.S." collapses to "us" and "plaintiff's" to "plaintiffs". */
+function isPunctuation(ch: string): boolean {
+  return !/[\p{L}\p{N}\s]/u.test(ch);
+}
+
 /**
  * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from
  * each character index in the normalized form back to the corresponding
- * index in the original text. Used by `findInDocumentContent` so matches
- * are tolerant of case + whitespace variance but can still return the
- * exact original excerpt.
+ * index in the original text. Used by `findInDocumentContent` (and server-side
+ * citation verification) so matches are tolerant of case + whitespace variance
+ * but can still return the exact original excerpt.
+ *
+ * With `stripPunctuation`, punctuation characters are removed from the
+ * normalized form too, making matching tolerant of punctuation drift (e.g. a
+ * model that adds a stray comma or drops a period). The index map still points
+ * back at the surviving original characters so the recovered excerpt is exact.
  */
-function normalizeWithMap(text: string): { norm: string; origIdx: number[] } {
+export function normalizeWithMap(
+  text: string,
+  opts: { stripPunctuation?: boolean } = {},
+): { norm: string; origIdx: number[] } {
+  const stripPunctuation = opts.stripPunctuation ?? false;
   const norm: string[] = [];
   const origIdx: number[] = [];
   let prevSpace = false;
@@ -1597,6 +1613,10 @@ function normalizeWithMap(text: string): { norm: string; origIdx: number[] } {
         origIdx.push(i);
         prevSpace = true;
       }
+    } else if (stripPunctuation && isPunctuation(ch)) {
+      // Drop punctuation without disturbing the space-collapsing state so
+      // "foo, bar" -> "foo bar" but "U.S." -> "us".
+      continue;
     } else {
       norm.push(ch.toLowerCase());
       origIdx.push(i);

backend/src/lib/chat/types.ts

@@ -57,6 +57,32 @@ export type ChatMessage = {
   workflow?: { id: string; title: string };
 };
 
+/**
+ * Result of server-side verification of a document quote against the extracted
+ * source text.
+ * - `verified`   — the quote was found and is character-identical to the source.
+ * - `repaired`   — the quote was found under whitespace/case/punctuation-tolerant
+ *                  matching but drifted from the source; `source_excerpt` holds the
+ *                  exact source text and has been swapped into the displayed quote.
+ * - `unverified` — no tolerant match; the model's quote is preserved but untrusted.
+ *
+ * A MISSING status (undefined) must be treated as untrusted by the UI — some
+ * paths (tabular, abort/error persistence) do not run verification.
+ */
+export type CitationVerificationStatus = "verified" | "unverified" | "repaired";
+
+/**
+ * Per-quote verification result. `start_char`/`end_char` index into the
+ * EXTRACTED source text (not the raw file bytes) and are only present for
+ * single-segment quotes that matched.
+ */
+export type QuoteVerification = {
+  status: CitationVerificationStatus;
+  start_char?: number;
+  end_char?: number;
+  source_excerpt?: string;
+};
+
 // ---------------------------------------------------------------------------
 // Doc resolution helpers (used by citations + documentOps)
 // ---------------------------------------------------------------------------

backend/src/lib/chat/verifyCitations.test.ts

@@ -0,0 +1,202 @@
+import { describe, it, expect, vi } from "vitest";
+
+// verifyCitations only reuses the pure normalizeWithMap matcher from
+// documentOps, but importing documentOps pulls in its storage/supabase graph.
+// Keep those module side-effects offline — this test injects source text
+// directly and never touches storage, proving verification adds no egress
+// (air-gap safe).
+vi.mock("../supabase", () => ({ createServerSupabase: vi.fn() }));
+vi.mock("../storage", () => ({ downloadFile: vi.fn() }));
+
+import {
+  locateQuote,
+  verifyQuoteAgainstSource,
+  verifyDocumentCitationAnnotation,
+  verifyDocumentCitations,
+} from "./verifyCitations";
+
+// Deterministic in-memory source text — no storage/model/network. Proves
+// verification only reads bytes handed to it (air-gap safe).
+const SOURCE = [
+  "## Page 1",
+  "The Tenant shall pay rent on the first day of each month.",
+  "The Landlord may terminate this Lease upon written notice.",
+].join("\n");
+
+function fetcherFor(map: Record<string, string>) {
+  return async (docId: string) => map[docId] ?? "";
+}
+
+function docAnnotation(quotes: { page: number | string; quote: string }[]) {
+  return {
+    type: "citation_data",
+    kind: "document" as const,
+    ref: 1,
+    doc_id: "doc-1",
+    document_id: "uuid-1",
+    filename: "lease.pdf",
+    page: quotes[0]?.page ?? 1,
+    quote: quotes[0]?.quote ?? "",
+    quotes,
+  };
+}
+
+describe("locateQuote", () => {
+  it("returns exact offsets for an exact substring match", () => {
+    const quote = "pay rent on the first day";
+    const loc = locateQuote(SOURCE, quote);
+    expect(loc).not.toBeNull();
+    expect(SOURCE.slice(loc!.start, loc!.end)).toBe(quote);
+    expect(loc!.excerpt).toBe(quote);
+  });
+
+  it("returns null when the quote is absent", () => {
+    expect(locateQuote(SOURCE, "the Tenant shall vacate immediately")).toBeNull();
+  });
+});
+
+describe("verifyQuoteAgainstSource", () => {
+  it("exact match → verified with correct offsets into the source", () => {
+    const quote = "The Landlord may terminate this Lease";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.status).toBe("verified");
+    expect(v.source_excerpt).toBe(quote);
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(quote);
+  });
+
+  it("whitespace + case drift → repaired with the exact source excerpt", () => {
+    const drifted = "the  landlord   MAY terminate this lease";
+    const v = verifyQuoteAgainstSource(SOURCE, drifted);
+    expect(v.status).toBe("repaired");
+    expect(v.source_excerpt).toBe("The Landlord may terminate this Lease");
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt);
+  });
+
+  it("punctuation drift → repaired with corrected excerpt and offsets", () => {
+    // Model inserted a stray comma the source does not contain.
+    const drifted = "pay rent, on the first day";
+    const v = verifyQuoteAgainstSource(SOURCE, drifted);
+    expect(v.status).toBe("repaired");
+    expect(v.source_excerpt).toBe("pay rent on the first day");
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt);
+  });
+
+  it("fabricated quote → unverified with no offsets", () => {
+    const v = verifyQuoteAgainstSource(SOURCE, "The Tenant waives all rights.");
+    expect(v.status).toBe("unverified");
+    expect(v.start_char).toBeUndefined();
+    expect(v.end_char).toBeUndefined();
+    expect(v.source_excerpt).toBeUndefined();
+  });
+
+  it("empty / unreadable source → unverified", () => {
+    expect(verifyQuoteAgainstSource("", "anything").status).toBe("unverified");
+    expect(
+      verifyQuoteAgainstSource("Document could not be read.", "anything").status,
+    ).toBe("unverified");
+  });
+
+  it("cross-page [[PAGE_BREAK]] quote → each segment verified independently", () => {
+    const src = "the first day of each month. The Landlord may terminate";
+    const quote = "the first day of each month[[PAGE_BREAK]]The Landlord may terminate";
+    const v = verifyQuoteAgainstSource(src, quote);
+    expect(v.status).toBe("verified");
+    expect(v.source_excerpt).toContain("[[PAGE_BREAK]]");
+  });
+
+  it("cross-page quote with a missing segment → unverified", () => {
+    const quote = "the first day of each month[[PAGE_BREAK]]never appears here";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.status).toBe("unverified");
+  });
+});
+
+describe("verifyDocumentCitationAnnotation", () => {
+  const fetcher = fetcherFor({ "doc-1": SOURCE });
+
+  it("marks a verified quote and attaches per-quote offsets + aggregate status", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verification_status).toBe("verified");
+    const quotes = ann.quotes as { verification: { status: string } }[];
+    expect(quotes[0].verification.status).toBe("verified");
+  });
+
+  it("repairs a drifted quote by swapping in the exact source excerpt", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "the  TENANT shall pay rent" }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verification_status).toBe("repaired");
+    const quotes = ann.quotes as {
+      quote: string;
+      verification: { status: string; source_excerpt: string };
+    }[];
+    expect(quotes[0].verification.status).toBe("repaired");
+    // The displayed quote is swapped to the true source text.
+    expect(quotes[0].quote).toBe("The Tenant shall pay rent");
+    // Legacy top-level quote mirror is updated too.
+    expect(ann.quote).toBe("The Tenant shall pay rent");
+  });
+
+  it("marks a fabricated quote unverified and preserves the model text", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant may sublet freely." }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verification_status).toBe("unverified");
+    const quotes = ann.quotes as { quote: string }[];
+    expect(quotes[0].quote).toBe("The Tenant may sublet freely.");
+  });
+
+  it("aggregates to unverified when any quote is unverified", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([
+        { page: 1, quote: "The Tenant shall pay rent" },
+        { page: 1, quote: "Nonexistent clause here." },
+      ]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verification_status).toBe("unverified");
+  });
+
+  it("unreadable source → all quotes unverified", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]),
+      fetcherFor({ "doc-1": "Document could not be read." }),
+    )) as Record<string, unknown>;
+    expect(ann.verification_status).toBe("unverified");
+  });
+
+  it("leaves case-law annotations untouched (no CourtListener regression)", async () => {
+    const caseAnn = {
+      type: "citation_data",
+      kind: "case",
+      ref: 2,
+      cluster_id: 42,
+      case_name: "Roe v. Doe",
+      quotes: [{ opinionId: null, type: null, author: null, quote: "held that…" }],
+    };
+    const out = await verifyDocumentCitationAnnotation(caseAnn, fetcher);
+    expect(out).toBe(caseAnn);
+    expect(out as Record<string, unknown>).not.toHaveProperty(
+      "verification_status",
+    );
+  });
+});
+
+describe("verifyDocumentCitations (batch)", () => {
+  it("verifies documents and passes case citations through unchanged", async () => {
+    const caseAnn = { type: "citation_data", kind: "case", ref: 2, cluster_id: 7 };
+    const out = await verifyDocumentCitations(
+      [docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), caseAnn],
+      fetcherFor({ "doc-1": SOURCE }),
+    );
+    expect((out[0] as Record<string, unknown>).verification_status).toBe(
+      "verified",
+    );
+    expect(out[1]).toBe(caseAnn);
+  });
+});