[Security 6/9] Server-side citation verification against source text

.gitignore

@@ -25,4 +25,5 @@ e2e/.auth/
 
 # Local Supabase scaffold (supabase init output) — the local stack is a test
 # harness only; CI scaffolds its own fresh copy on every run
+/supabase/
 backend/supabase/

backend/src/lib/chat/prompts.ts

@@ -35,6 +35,7 @@ Citation rules:
 - A cell tagged "⟨merged A1:C1⟩" spans that whole range: its value belongs to the anchor cell and the other covered cells are shown blank. When citing anything in a merged range, set "cell" to the full range from the tag (e.g. "A1:C1"), not a covered cell like "B1". Do not include the "⟨merged ...⟩" tag text in "quote".
 - For a continuous quote crossing two pages, set "page" to "N-M" and include [[PAGE_BREAK]] at the page break. Otherwise, use separate quote objects.
 - For legacy compatibility, you may also include top-level "page" and "quote" matching the first quote.
+- TESTING ONLY: Make 50% of document citation quotes exact source text and 50% deliberately false text that does not appear in the source, so citation verification produces both outcomes. Use an even number of document quotes. Do not alter case-law citations.
 - Omit the <CITATIONS> block when there are no citations.
 
 DOCX GENERATION:

backend/src/lib/chat/streaming.ts

@@ -25,6 +25,7 @@ import {
   type AskInputsEvent,
   type EditAnnotation,
   devLog,
+  resolveDocLabel,
 } from "./types";
 import { TOOLS, WORKFLOW_TOOLS } from "./tools/toolSchemas";
 import {
@@ -38,9 +39,11 @@ import {
   type CourtlistenerTurnState,
 } from "./tools/toolDispatcher";
 import {
+  readDocumentContent,
   type TurnEditState,
   type TurnReadState,
 } from "./tools/documentOps";
+import { verifyDocumentCitations } from "./verifyCitations";
 
 
 export type AssistantEvent =
@@ -527,15 +530,38 @@ export async function runLLMStream(params: {
   // Parse and emit citations from <CITATIONS> block
   const { citations: parsedCitations, diagnostics: citationDiagnostics } =
     parseCitationsWithDiagnostics(fullText);
-  const citations = buildCitations
-    ? buildCitations(fullText)
-    : parsedCitations.map((c) =>
-        createCitation(
-          c,
-          docIndex,
-          courtlistenerTurnState.casesByClusterId,
-        ),
-      );
+  let citations: unknown[];
+  if (buildCitations) {
+    // Custom builders (tabular) bypass document-citation verification.
+    citations = buildCitations(fullText);
+  } else {
+    const rawCitations = parsedCitations.map((c) =>
+      createCitation(
+        c,
+        docIndex,
+        courtlistenerTurnState.casesByClusterId,
+      ),
+    );
+    // Server-side document-quote verification. Fetch each document's extracted
+    // source text at most once per turn (memoized by doc_id), reading only the
+    // bytes already in storage with emitEvents:false so no new events fire and
+    // the air-gap guarantee holds. Case citations pass through untouched.
+    const sourceTextByDocId = new Map<string, Promise<string>>();
+    const getSourceText = (docId: string): Promise<string> => {
+      let pending = sourceTextByDocId.get(docId);
+      if (!pending) {
+        const label = resolveDocLabel(docId, docStore, docIndex);
+        pending = label
+          ? readDocumentContent(label, docStore, () => {}, docIndex, db, {
+              emitEvents: false,
+            })
+          : Promise.resolve("");
+        sourceTextByDocId.set(docId, pending);
+      }
+      return pending;
+    };
+    citations = await verifyDocumentCitations(rawCitations, getSourceText);
+  }
   devLog("[chat/stream] final citations", {
     hasCitationsBlock: citationDiagnostics.hasBlock,
     citationsBlockLength: citationDiagnostics.rawLength,

backend/src/lib/chat/tools/documentOps.ts

@@ -1578,14 +1578,30 @@ export async function readDocumentContent(
   }
 }
 
+/** A character is "punctuation" for tolerant matching if it is not a letter,
+ *  number, or whitespace. Dropped entirely (not replaced with a space) so
+ *  "U.S." collapses to "us" and "plaintiff's" to "plaintiffs". */
+function isPunctuation(ch: string): boolean {
+  return !/[\p{L}\p{N}\s]/u.test(ch);
+}
+
 /**
  * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from
  * each character index in the normalized form back to the corresponding
- * index in the original text. Used by `findInDocumentContent` so matches
- * are tolerant of case + whitespace variance but can still return the
- * exact original excerpt.
+ * index in the original text. Used by `findInDocumentContent` (and server-side
+ * citation verification) so matches are tolerant of case + whitespace variance
+ * but can still return the exact original excerpt.
+ *
+ * With `stripPunctuation`, punctuation characters are removed from the
+ * normalized form too, making matching tolerant of punctuation drift (e.g. a
+ * model that adds a stray comma or drops a period). The index map still points
+ * back at the surviving original characters so the recovered excerpt is exact.
  */
-function normalizeWithMap(text: string): { norm: string; origIdx: number[] } {
+export function normalizeWithMap(
+  text: string,
+  opts: { stripPunctuation?: boolean } = {},
+): { norm: string; origIdx: number[] } {
+  const stripPunctuation = opts.stripPunctuation ?? false;
   const norm: string[] = [];
   const origIdx: number[] = [];
   let prevSpace = false;
@@ -1597,6 +1613,10 @@ function normalizeWithMap(text: string): { norm: string; origIdx: number[] } {
         origIdx.push(i);
         prevSpace = true;
       }
+    } else if (stripPunctuation && isPunctuation(ch)) {
+      // Drop punctuation without disturbing the space-collapsing state so
+      // "foo, bar" -> "foo bar" but "U.S." -> "us".
+      continue;
     } else {
       norm.push(ch.toLowerCase());
       origIdx.push(i);

backend/src/lib/chat/types.ts

@@ -57,6 +57,18 @@ export type ChatMessage = {
   workflow?: { id: string; title: string };
 };
 
+/**
+ * Per-quote verification result. `start_char`/`end_char` index into the
+ * EXTRACTED source text (not the raw file bytes) and are only present for
+ * single-segment quotes that matched.
+ */
+export type QuoteVerification = {
+  verified: boolean;
+  start_char?: number;
+  end_char?: number;
+  source_excerpt?: string;
+};
+
 // ---------------------------------------------------------------------------
 // Doc resolution helpers (used by citations + documentOps)
 // ---------------------------------------------------------------------------

backend/src/lib/chat/verifyCitations.test.ts

@@ -0,0 +1,243 @@
+import { describe, it, expect, vi } from "vitest";
+
+// verifyCitations only reuses the pure normalizeWithMap matcher from
+// documentOps, but importing documentOps pulls in its storage/supabase graph.
+// Keep those module side-effects offline — this test injects source text
+// directly and never touches storage, proving verification adds no egress
+// (air-gap safe).
+vi.mock("../supabase", () => ({ createServerSupabase: vi.fn() }));
+vi.mock("../storage", () => ({ downloadFile: vi.fn() }));
+
+import {
+  locateQuote,
+  verifyQuoteAgainstSource,
+  verifyDocumentCitationAnnotation,
+  verifyDocumentCitations,
+} from "./verifyCitations";
+
+// Deterministic in-memory source text — no storage/model/network. Proves
+// verification only reads bytes handed to it (air-gap safe).
+const SOURCE = [
+  "## Page 1",
+  "The Tenant shall pay rent on the first day of each month.",
+  "The Landlord may terminate this Lease upon written notice.",
+].join("\n");
+
+function fetcherFor(map: Record<string, string>) {
+  return async (docId: string) => map[docId] ?? "";
+}
+
+function docAnnotation(quotes: { page: number | string; quote: string }[]) {
+  return {
+    type: "citation_data",
+    kind: "document" as const,
+    ref: 1,
+    doc_id: "doc-1",
+    document_id: "uuid-1",
+    filename: "lease.pdf",
+    page: quotes[0]?.page ?? 1,
+    quote: quotes[0]?.quote ?? "",
+    quotes,
+  };
+}
+
+describe("locateQuote", () => {
+  it("returns exact offsets for an exact substring match", () => {
+    const quote = "pay rent on the first day";
+    const loc = locateQuote(SOURCE, quote);
+    expect(loc).not.toBeNull();
+    expect(SOURCE.slice(loc!.start, loc!.end)).toBe(quote);
+    expect(loc!.excerpt).toBe(quote);
+  });
+
+  it("returns null when the quote is absent", () => {
+    expect(
+      locateQuote(SOURCE, "the Tenant shall vacate immediately"),
+    ).toBeNull();
+  });
+});
+
+describe("verifyQuoteAgainstSource", () => {
+  it("exact match → verified with correct offsets into the source", () => {
+    const quote = "The Landlord may terminate this Lease";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.verified).toBe(true);
+    expect(v.needs_correction).toBe(false);
+    expect(v.source_excerpt).toBe(quote);
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(quote);
+  });
+
+  it("whitespace + case drift → verified with the exact source excerpt", () => {
+    const drifted = "the  landlord   MAY terminate this lease";
+    const v = verifyQuoteAgainstSource(SOURCE, drifted);
+    expect(v.verified).toBe(true);
+    expect(v.needs_correction).toBe(true);
+    expect(v.source_excerpt).toBe("The Landlord may terminate this Lease");
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt);
+  });
+
+  it("punctuation drift → verified with corrected excerpt and offsets", () => {
+    // Model inserted a stray comma the source does not contain.
+    const drifted = "pay rent, on the first day";
+    const v = verifyQuoteAgainstSource(SOURCE, drifted);
+    expect(v.verified).toBe(true);
+    expect(v.needs_correction).toBe(true);
+    expect(v.source_excerpt).toBe("pay rent on the first day");
+    expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt);
+  });
+
+  it("fabricated quote → unverified with no offsets", () => {
+    const v = verifyQuoteAgainstSource(SOURCE, "The Tenant waives all rights.");
+    expect(v.verified).toBe(false);
+    expect(v.start_char).toBeUndefined();
+    expect(v.end_char).toBeUndefined();
+    expect(v.source_excerpt).toBeUndefined();
+  });
+
+  it("empty / unreadable source → unverified", () => {
+    expect(verifyQuoteAgainstSource("", "anything").verified).toBe(false);
+    expect(
+      verifyQuoteAgainstSource("Document could not be read.", "anything")
+        .verified,
+    ).toBe(false);
+  });
+
+  it("cross-page [[PAGE_BREAK]] quote → each segment verified independently", () => {
+    const src = "the first day of each month. The Landlord may terminate";
+    const quote =
+      "the first day of each month[[PAGE_BREAK]]The Landlord may terminate";
+    const v = verifyQuoteAgainstSource(src, quote);
+    expect(v.verified).toBe(true);
+    expect(v.source_excerpt).toContain("[[PAGE_BREAK]]");
+  });
+
+  it("cross-page quote with a missing segment → unverified", () => {
+    const quote = "the first day of each month[[PAGE_BREAK]]never appears here";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.verified).toBe(false);
+  });
+
+  it.each(["...", "…"])(
+    "ellipsis-separated quote using %s verifies each excerpt independently",
+    (ellipsis) => {
+      const quote = `The Tenant shall pay rent${ellipsis}The Landlord may terminate this Lease`;
+      const v = verifyQuoteAgainstSource(SOURCE, quote);
+      expect(v.verified).toBe(true);
+      expect(v.source_excerpt).toBe(
+        "The Tenant shall pay rent ... The Landlord may terminate this Lease",
+      );
+    },
+  );
+
+  it("ellipsis-separated quote with a missing excerpt → unverified", () => {
+    const quote =
+      "The Tenant shall pay rent...the Tenant may sublet without consent";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.verified).toBe(false);
+  });
+
+  it("supports ellipsis omissions inside a cross-page quote", () => {
+    const quote =
+      "The Tenant shall pay...first day of each month[[PAGE_BREAK]]The Landlord may terminate...written notice";
+    const v = verifyQuoteAgainstSource(SOURCE, quote);
+    expect(v.verified).toBe(true);
+    expect(v.source_excerpt).toContain("[[PAGE_BREAK]]");
+  });
+});
+
+describe("verifyDocumentCitationAnnotation", () => {
+  const fetcher = fetcherFor({ "doc-1": SOURCE });
+
+  it("marks a verified quote and attaches per-quote offsets", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verified).toBe(true);
+    const quotes = ann.quotes as { verification: { verified: boolean } }[];
+    expect(quotes[0].verification.verified).toBe(true);
+  });
+
+  it("corrects a drifted quote by swapping in the exact source excerpt", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "the  TENANT shall pay rent" }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verified).toBe(true);
+    const quotes = ann.quotes as {
+      quote: string;
+      verification: { verified: boolean; source_excerpt: string };
+    }[];
+    expect(quotes[0].verification.verified).toBe(true);
+    expect(quotes[0].verification).not.toHaveProperty("needs_correction");
+    // The displayed quote is swapped to the true source text.
+    expect(quotes[0].quote).toBe("The Tenant shall pay rent");
+    // Legacy top-level quote mirror is updated too.
+    expect(ann.quote).toBe("The Tenant shall pay rent");
+  });
+
+  it("marks a fabricated quote unverified and preserves the model text", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant may sublet freely." }]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verified).toBe(false);
+    const quotes = ann.quotes as { quote: string }[];
+    expect(quotes[0].quote).toBe("The Tenant may sublet freely.");
+  });
+
+  it("aggregates to unverified when any quote is unverified", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([
+        { page: 1, quote: "The Tenant shall pay rent" },
+        { page: 1, quote: "Nonexistent clause here." },
+      ]),
+      fetcher,
+    )) as Record<string, unknown>;
+    expect(ann.verified).toBe(false);
+  });
+
+  it("unreadable source → all quotes unverified", async () => {
+    const ann = (await verifyDocumentCitationAnnotation(
+      docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]),
+      fetcherFor({ "doc-1": "Document could not be read." }),
+    )) as Record<string, unknown>;
+    expect(ann.verified).toBe(false);
+  });
+
+  it("leaves case-law annotations untouched (no CourtListener regression)", async () => {
+    const caseAnn = {
+      type: "citation_data",
+      kind: "case",
+      ref: 2,
+      cluster_id: 42,
+      case_name: "Roe v. Doe",
+      quotes: [
+        { opinionId: null, type: null, author: null, quote: "held that…" },
+      ],
+    };
+    const out = await verifyDocumentCitationAnnotation(caseAnn, fetcher);
+    expect(out).toBe(caseAnn);
+    expect(out as Record<string, unknown>).not.toHaveProperty("verified");
+  });
+});
+
+describe("verifyDocumentCitations (batch)", () => {
+  it("verifies documents and passes case citations through unchanged", async () => {
+    const caseAnn = {
+      type: "citation_data",
+      kind: "case",
+      ref: 2,
+      cluster_id: 7,
+    };
+    const out = await verifyDocumentCitations(
+      [
+        docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]),
+        caseAnn,
+      ],
+      fetcherFor({ "doc-1": SOURCE }),
+    );
+    expect((out[0] as Record<string, unknown>).verified).toBe(true);
+    expect(out[1]).toBe(caseAnn);
+  });
+});