From a05e2956920fc54f17c526ff32d995e076975a61 Mon Sep 17 00:00:00 2001 From: Amalanand Muthukumaran Date: Sat, 25 Jul 2026 14:30:37 -0700 Subject: [PATCH 1/3] feat: server-side citation verification against source text Port of the fork's document-quote verification (apps/api/src/lib/tools/ verifyCitations.ts and its wiring) to upstream layout. After the model's block is parsed, each document quote is located in the document's extracted source text (exact, then whitespace/case-tolerant, then punctuation-tolerant matching). Quotes get a per-quote verification record and each citation an aggregate verification_status: verified | repaired (exact source excerpt swapped in) | unverified. Case-law citations pass through untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01CEguyEgXa9JjCciXCcVemC --- backend/src/lib/chat/streaming.ts | 45 ++++- backend/src/lib/chat/tools/documentOps.ts | 28 ++- backend/src/lib/chat/types.ts | 26 +++ backend/src/lib/chat/verifyCitations.test.ts | 202 +++++++++++++++++++ backend/src/lib/chat/verifyCitations.ts | 185 +++++++++++++++++ 5 files changed, 473 insertions(+), 13 deletions(-) create mode 100644 backend/src/lib/chat/verifyCitations.test.ts create mode 100644 backend/src/lib/chat/verifyCitations.ts diff --git a/backend/src/lib/chat/streaming.ts b/backend/src/lib/chat/streaming.ts index f6ddacb2d..41f175856 100644 --- a/backend/src/lib/chat/streaming.ts +++ b/backend/src/lib/chat/streaming.ts @@ -25,6 +25,7 @@ import { type AskInputsEvent, type EditAnnotation, devLog, + resolveDocLabel, } from "./types"; import { TOOLS, WORKFLOW_TOOLS } from "./tools/toolSchemas"; import { @@ -38,9 +39,11 @@ import { type CourtlistenerTurnState, } from "./tools/toolDispatcher"; import { + readDocumentContent, type TurnEditState, type TurnReadState, } from "./tools/documentOps"; +import { verifyDocumentCitations } from "./verifyCitations"; export type AssistantEvent = @@ -527,15 +530,39 @@ export async function runLLMStream(params: { // Parse and emit citations from block const { citations: parsedCitations, diagnostics: citationDiagnostics } = parseCitationsWithDiagnostics(fullText); - const citations = buildCitations - ? buildCitations(fullText) - : parsedCitations.map((c) => - createCitation( - c, - docIndex, - courtlistenerTurnState.casesByClusterId, - ), - ); + let citations: unknown[]; + if (buildCitations) { + // Custom builders (tabular) bypass verification; annotations carry no + // verification_status and the UI treats a missing status as untrusted. + citations = buildCitations(fullText); + } else { + const rawCitations = parsedCitations.map((c) => + createCitation( + c, + docIndex, + courtlistenerTurnState.casesByClusterId, + ), + ); + // Server-side document-quote verification. Fetch each document's extracted + // source text at most once per turn (memoized by doc_id), reading only the + // bytes already in storage with emitEvents:false so no new events fire and + // the air-gap guarantee holds. Case citations pass through untouched. + const sourceTextByDocId = new Map>(); + const getSourceText = (docId: string): Promise => { + let pending = sourceTextByDocId.get(docId); + if (!pending) { + const label = resolveDocLabel(docId, docStore, docIndex); + pending = label + ? readDocumentContent(label, docStore, () => {}, docIndex, db, { + emitEvents: false, + }) + : Promise.resolve(""); + sourceTextByDocId.set(docId, pending); + } + return pending; + }; + citations = await verifyDocumentCitations(rawCitations, getSourceText); + } devLog("[chat/stream] final citations", { hasCitationsBlock: citationDiagnostics.hasBlock, citationsBlockLength: citationDiagnostics.rawLength, diff --git a/backend/src/lib/chat/tools/documentOps.ts b/backend/src/lib/chat/tools/documentOps.ts index 1ca2a5546..3b8b9dc49 100644 --- a/backend/src/lib/chat/tools/documentOps.ts +++ b/backend/src/lib/chat/tools/documentOps.ts @@ -1578,14 +1578,30 @@ export async function readDocumentContent( } } +/** A character is "punctuation" for tolerant matching if it is not a letter, + * number, or whitespace. Dropped entirely (not replaced with a space) so + * "U.S." collapses to "us" and "plaintiff's" to "plaintiffs". */ +function isPunctuation(ch: string): boolean { + return !/[\p{L}\p{N}\s]/u.test(ch); +} + /** * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from * each character index in the normalized form back to the corresponding - * index in the original text. Used by `findInDocumentContent` so matches - * are tolerant of case + whitespace variance but can still return the - * exact original excerpt. + * index in the original text. Used by `findInDocumentContent` (and server-side + * citation verification) so matches are tolerant of case + whitespace variance + * but can still return the exact original excerpt. + * + * With `stripPunctuation`, punctuation characters are removed from the + * normalized form too, making matching tolerant of punctuation drift (e.g. a + * model that adds a stray comma or drops a period). The index map still points + * back at the surviving original characters so the recovered excerpt is exact. */ -function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { +export function normalizeWithMap( + text: string, + opts: { stripPunctuation?: boolean } = {}, +): { norm: string; origIdx: number[] } { + const stripPunctuation = opts.stripPunctuation ?? false; const norm: string[] = []; const origIdx: number[] = []; let prevSpace = false; @@ -1597,6 +1613,10 @@ function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { origIdx.push(i); prevSpace = true; } + } else if (stripPunctuation && isPunctuation(ch)) { + // Drop punctuation without disturbing the space-collapsing state so + // "foo, bar" -> "foo bar" but "U.S." -> "us". + continue; } else { norm.push(ch.toLowerCase()); origIdx.push(i); diff --git a/backend/src/lib/chat/types.ts b/backend/src/lib/chat/types.ts index 60c10fba6..12a87fb0d 100644 --- a/backend/src/lib/chat/types.ts +++ b/backend/src/lib/chat/types.ts @@ -57,6 +57,32 @@ export type ChatMessage = { workflow?: { id: string; title: string }; }; +/** + * Result of server-side verification of a document quote against the extracted + * source text. + * - `verified` — the quote was found and is character-identical to the source. + * - `repaired` — the quote was found under whitespace/case/punctuation-tolerant + * matching but drifted from the source; `source_excerpt` holds the + * exact source text and has been swapped into the displayed quote. + * - `unverified` — no tolerant match; the model's quote is preserved but untrusted. + * + * A MISSING status (undefined) must be treated as untrusted by the UI — some + * paths (tabular, abort/error persistence) do not run verification. + */ +export type CitationVerificationStatus = "verified" | "unverified" | "repaired"; + +/** + * Per-quote verification result. `start_char`/`end_char` index into the + * EXTRACTED source text (not the raw file bytes) and are only present for + * single-segment quotes that matched. + */ +export type QuoteVerification = { + status: CitationVerificationStatus; + start_char?: number; + end_char?: number; + source_excerpt?: string; +}; + // --------------------------------------------------------------------------- // Doc resolution helpers (used by citations + documentOps) // --------------------------------------------------------------------------- diff --git a/backend/src/lib/chat/verifyCitations.test.ts b/backend/src/lib/chat/verifyCitations.test.ts new file mode 100644 index 000000000..78f285b26 --- /dev/null +++ b/backend/src/lib/chat/verifyCitations.test.ts @@ -0,0 +1,202 @@ +import { describe, it, expect, vi } from "vitest"; + +// verifyCitations only reuses the pure normalizeWithMap matcher from +// documentOps, but importing documentOps pulls in its storage/supabase graph. +// Keep those module side-effects offline — this test injects source text +// directly and never touches storage, proving verification adds no egress +// (air-gap safe). +vi.mock("../supabase", () => ({ createServerSupabase: vi.fn() })); +vi.mock("../storage", () => ({ downloadFile: vi.fn() })); + +import { + locateQuote, + verifyQuoteAgainstSource, + verifyDocumentCitationAnnotation, + verifyDocumentCitations, +} from "./verifyCitations"; + +// Deterministic in-memory source text — no storage/model/network. Proves +// verification only reads bytes handed to it (air-gap safe). +const SOURCE = [ + "## Page 1", + "The Tenant shall pay rent on the first day of each month.", + "The Landlord may terminate this Lease upon written notice.", +].join("\n"); + +function fetcherFor(map: Record) { + return async (docId: string) => map[docId] ?? ""; +} + +function docAnnotation(quotes: { page: number | string; quote: string }[]) { + return { + type: "citation_data", + kind: "document" as const, + ref: 1, + doc_id: "doc-1", + document_id: "uuid-1", + filename: "lease.pdf", + page: quotes[0]?.page ?? 1, + quote: quotes[0]?.quote ?? "", + quotes, + }; +} + +describe("locateQuote", () => { + it("returns exact offsets for an exact substring match", () => { + const quote = "pay rent on the first day"; + const loc = locateQuote(SOURCE, quote); + expect(loc).not.toBeNull(); + expect(SOURCE.slice(loc!.start, loc!.end)).toBe(quote); + expect(loc!.excerpt).toBe(quote); + }); + + it("returns null when the quote is absent", () => { + expect(locateQuote(SOURCE, "the Tenant shall vacate immediately")).toBeNull(); + }); +}); + +describe("verifyQuoteAgainstSource", () => { + it("exact match → verified with correct offsets into the source", () => { + const quote = "The Landlord may terminate this Lease"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.status).toBe("verified"); + expect(v.source_excerpt).toBe(quote); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(quote); + }); + + it("whitespace + case drift → repaired with the exact source excerpt", () => { + const drifted = "the landlord MAY terminate this lease"; + const v = verifyQuoteAgainstSource(SOURCE, drifted); + expect(v.status).toBe("repaired"); + expect(v.source_excerpt).toBe("The Landlord may terminate this Lease"); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); + }); + + it("punctuation drift → repaired with corrected excerpt and offsets", () => { + // Model inserted a stray comma the source does not contain. + const drifted = "pay rent, on the first day"; + const v = verifyQuoteAgainstSource(SOURCE, drifted); + expect(v.status).toBe("repaired"); + expect(v.source_excerpt).toBe("pay rent on the first day"); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); + }); + + it("fabricated quote → unverified with no offsets", () => { + const v = verifyQuoteAgainstSource(SOURCE, "The Tenant waives all rights."); + expect(v.status).toBe("unverified"); + expect(v.start_char).toBeUndefined(); + expect(v.end_char).toBeUndefined(); + expect(v.source_excerpt).toBeUndefined(); + }); + + it("empty / unreadable source → unverified", () => { + expect(verifyQuoteAgainstSource("", "anything").status).toBe("unverified"); + expect( + verifyQuoteAgainstSource("Document could not be read.", "anything").status, + ).toBe("unverified"); + }); + + it("cross-page [[PAGE_BREAK]] quote → each segment verified independently", () => { + const src = "the first day of each month. The Landlord may terminate"; + const quote = "the first day of each month[[PAGE_BREAK]]The Landlord may terminate"; + const v = verifyQuoteAgainstSource(src, quote); + expect(v.status).toBe("verified"); + expect(v.source_excerpt).toContain("[[PAGE_BREAK]]"); + }); + + it("cross-page quote with a missing segment → unverified", () => { + const quote = "the first day of each month[[PAGE_BREAK]]never appears here"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.status).toBe("unverified"); + }); +}); + +describe("verifyDocumentCitationAnnotation", () => { + const fetcher = fetcherFor({ "doc-1": SOURCE }); + + it("marks a verified quote and attaches per-quote offsets + aggregate status", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + fetcher, + )) as Record; + expect(ann.verification_status).toBe("verified"); + const quotes = ann.quotes as { verification: { status: string } }[]; + expect(quotes[0].verification.status).toBe("verified"); + }); + + it("repairs a drifted quote by swapping in the exact source excerpt", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "the TENANT shall pay rent" }]), + fetcher, + )) as Record; + expect(ann.verification_status).toBe("repaired"); + const quotes = ann.quotes as { + quote: string; + verification: { status: string; source_excerpt: string }; + }[]; + expect(quotes[0].verification.status).toBe("repaired"); + // The displayed quote is swapped to the true source text. + expect(quotes[0].quote).toBe("The Tenant shall pay rent"); + // Legacy top-level quote mirror is updated too. + expect(ann.quote).toBe("The Tenant shall pay rent"); + }); + + it("marks a fabricated quote unverified and preserves the model text", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant may sublet freely." }]), + fetcher, + )) as Record; + expect(ann.verification_status).toBe("unverified"); + const quotes = ann.quotes as { quote: string }[]; + expect(quotes[0].quote).toBe("The Tenant may sublet freely."); + }); + + it("aggregates to unverified when any quote is unverified", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([ + { page: 1, quote: "The Tenant shall pay rent" }, + { page: 1, quote: "Nonexistent clause here." }, + ]), + fetcher, + )) as Record; + expect(ann.verification_status).toBe("unverified"); + }); + + it("unreadable source → all quotes unverified", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + fetcherFor({ "doc-1": "Document could not be read." }), + )) as Record; + expect(ann.verification_status).toBe("unverified"); + }); + + it("leaves case-law annotations untouched (no CourtListener regression)", async () => { + const caseAnn = { + type: "citation_data", + kind: "case", + ref: 2, + cluster_id: 42, + case_name: "Roe v. Doe", + quotes: [{ opinionId: null, type: null, author: null, quote: "held that…" }], + }; + const out = await verifyDocumentCitationAnnotation(caseAnn, fetcher); + expect(out).toBe(caseAnn); + expect(out as Record).not.toHaveProperty( + "verification_status", + ); + }); +}); + +describe("verifyDocumentCitations (batch)", () => { + it("verifies documents and passes case citations through unchanged", async () => { + const caseAnn = { type: "citation_data", kind: "case", ref: 2, cluster_id: 7 }; + const out = await verifyDocumentCitations( + [docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), caseAnn], + fetcherFor({ "doc-1": SOURCE }), + ); + expect((out[0] as Record).verification_status).toBe( + "verified", + ); + expect(out[1]).toBe(caseAnn); + }); +}); diff --git a/backend/src/lib/chat/verifyCitations.ts b/backend/src/lib/chat/verifyCitations.ts new file mode 100644 index 000000000..fb0d49afa --- /dev/null +++ b/backend/src/lib/chat/verifyCitations.ts @@ -0,0 +1,185 @@ +import type { + CitationVerificationStatus, + QuoteVerification, +} from "./types"; +import { normalizeWithMap } from "./tools/documentOps"; + +// Mirrors the frontend: cross-page quotes join two page segments with this +// sentinel (see expandDocumentQuoteEntry in +// frontend/src/app/components/shared/types.ts). +const PAGE_BREAK_SENTINEL = "[[PAGE_BREAK]]"; + +// Source-text sentinels returned by readDocumentContent when a document can't +// be read. Treat these as "no source" so every quote falls back to unverified +// rather than false-negative matching against the literal error string. +const UNREADABLE_SOURCES = new Set([ + "Document could not be read.", + "Document not found.", +]); + +type QuoteLocation = { start: number; end: number; excerpt: string }; + +/** + * Locate `quote` inside `source`, returning the exact original substring + * (`excerpt`) plus its char offsets into `source`. Tries progressively more + * tolerant matchers and returns the first hit: + * 1. exact substring + * 2. whitespace + case normalized + * 3. whitespace + case + punctuation normalized (tolerant/fuzzy) + * Offsets index into the EXTRACTED source text, not the raw file bytes. + */ +export function locateQuote(source: string, quote: string): QuoteLocation | null { + if (!source || !quote) return null; + + // Tier 1: exact. + const exactIdx = source.indexOf(quote); + if (exactIdx >= 0) { + return { start: exactIdx, end: exactIdx + quote.length, excerpt: quote }; + } + + // Tier 2: whitespace + case. Tier 3: also punctuation-tolerant. + return ( + locateNormalized(source, quote, {}) ?? + locateNormalized(source, quote, { stripPunctuation: true }) + ); +} + +function locateNormalized( + source: string, + quote: string, + opts: { stripPunctuation?: boolean }, +): QuoteLocation | null { + const { norm, origIdx } = normalizeWithMap(source, opts); + const needle = normalizeWithMap(quote, opts).norm.trim(); + if (!needle) return null; + const pos = norm.indexOf(needle); + if (pos < 0) return null; + const endNormPos = pos + needle.length; + const start = origIdx[pos] ?? 0; + const end = + endNormPos - 1 < origIdx.length + ? origIdx[endNormPos - 1] + 1 + : source.length; + return { start, end, excerpt: source.slice(start, end) }; +} + +/** + * Verify a single model quote against the source text, returning the + * per-quote verification record. Cross-page quotes (containing the + * `[[PAGE_BREAK]]` sentinel) are split and each segment verified independently; + * char offsets are only attached for single-segment quotes. + */ +export function verifyQuoteAgainstSource( + source: string, + quote: string, +): QuoteVerification { + if (!source || UNREADABLE_SOURCES.has(source)) { + return { status: "unverified" }; + } + + if (quote.includes(PAGE_BREAK_SENTINEL)) { + const segments = quote + .split(PAGE_BREAK_SENTINEL) + .map((s) => s.trim()) + .filter((s) => s.length > 0); + if (!segments.length) return { status: "unverified" }; + const located = segments.map((seg) => ({ seg, loc: locateQuote(source, seg) })); + if (located.some((l) => !l.loc)) return { status: "unverified" }; + const anyRepaired = located.some((l) => l.loc!.excerpt !== l.seg); + return { + status: anyRepaired ? "repaired" : "verified", + source_excerpt: located + .map((l) => l.loc!.excerpt) + .join(` ${PAGE_BREAK_SENTINEL} `), + }; + } + + const loc = locateQuote(source, quote); + if (!loc) return { status: "unverified" }; + return { + status: loc.excerpt === quote ? "verified" : "repaired", + start_char: loc.start, + end_char: loc.end, + source_excerpt: loc.excerpt, + }; +} + +/** Aggregate a list of per-quote statuses: unverified beats repaired beats verified. */ +function aggregateStatus( + statuses: CitationVerificationStatus[], +): CitationVerificationStatus { + if (statuses.some((s) => s === "unverified")) return "unverified"; + if (statuses.some((s) => s === "repaired")) return "repaired"; + return "verified"; +} + +type DocQuoteEntry = { page: number | string; quote: string }; + +/** + * Attach server-side verification to one document citation annotation. + * Case-law annotations (kind === "case") are returned untouched — their + * existence is verified upstream via CourtListener and must never be + * re-marked. For document annotations, source text is fetched once via + * `getSourceText(doc_id)` and each quote is located in it; repaired quotes + * have the exact source excerpt swapped in so the UI never shows drifted text. + */ +export async function verifyDocumentCitationAnnotation( + annotation: unknown, + getSourceText: (docId: string) => Promise, +): Promise { + if (!annotation || typeof annotation !== "object") return annotation; + const a = annotation as Record; + if (a.kind === "case") return annotation; + const docId = typeof a.doc_id === "string" ? a.doc_id : null; + if (!docId) return annotation; + + const entries: DocQuoteEntry[] = Array.isArray(a.quotes) + ? (a.quotes as DocQuoteEntry[]) + : typeof a.quote === "string" + ? [{ page: (a.page as number | string) ?? 1, quote: a.quote }] + : []; + if (!entries.length) return annotation; + + let source: string; + try { + source = await getSourceText(docId); + } catch { + source = ""; + } + + const verifiedQuotes = entries.map((entry) => { + const verification = verifyQuoteAgainstSource(source, entry.quote); + // Swap the exact source text into the displayed quote when we repaired it, + // so a drifted quote is never surfaced as the source's words. + const quote = + verification.status === "repaired" && verification.source_excerpt + ? verification.source_excerpt + : entry.quote; + return { ...entry, quote, verification }; + }); + + const verificationStatus = aggregateStatus( + verifiedQuotes.map((q) => q.verification.status), + ); + + return { + ...a, + quote: verifiedQuotes[0]?.quote ?? a.quote, + quotes: verifiedQuotes, + verification_status: verificationStatus, + }; +} + +/** + * Verify a batch of citation annotations. Document annotations are verified + * against source text supplied by `getSourceText`; case annotations pass + * through unchanged. + */ +export async function verifyDocumentCitations( + annotations: unknown[], + getSourceText: (docId: string) => Promise, +): Promise { + return Promise.all( + annotations.map((a) => verifyDocumentCitationAnnotation(a, getSourceText)), + ); +} From 2fc7a6da5dd7809ceb6edf7bbe3ec1d73a5465d1 Mon Sep 17 00:00:00 2001 From: willchen96 Date: Tue, 4 Aug 2026 19:18:38 +0800 Subject: [PATCH 2/3] feat: surface unverified citation quotes --- backend/src/lib/chat/prompts.ts | 1 + backend/src/lib/chat/streaming.ts | 3 +- backend/src/lib/chat/types.ts | 16 +-- backend/src/lib/chat/verifyCitations.test.ts | 107 ++++++++++++------ backend/src/lib/chat/verifyCitations.ts | 101 +++++++++++------ .../assistant/CitationQuotesHeader.tsx | 70 ++++++++---- .../src/app/components/assistant/DocPanel.tsx | 24 +++- .../message/CitationSources.test.tsx | 62 ++++++++++ .../assistant/message/CitationSources.tsx | 17 ++- .../assistant/message/MarkdownContent.tsx | 11 +- .../message/citationVerification.test.tsx | 107 ++++++++++++++++++ .../message/citationVerification.tsx | 78 +++++++++++++ frontend/src/app/components/shared/types.ts | 10 ++ 13 files changed, 493 insertions(+), 114 deletions(-) create mode 100644 frontend/src/app/components/assistant/message/CitationSources.test.tsx create mode 100644 frontend/src/app/components/assistant/message/citationVerification.test.tsx create mode 100644 frontend/src/app/components/assistant/message/citationVerification.tsx diff --git a/backend/src/lib/chat/prompts.ts b/backend/src/lib/chat/prompts.ts index 6b3f2fc3d..b56ac14aa 100644 --- a/backend/src/lib/chat/prompts.ts +++ b/backend/src/lib/chat/prompts.ts @@ -35,6 +35,7 @@ Citation rules: - A cell tagged "⟨merged A1:C1⟩" spans that whole range: its value belongs to the anchor cell and the other covered cells are shown blank. When citing anything in a merged range, set "cell" to the full range from the tag (e.g. "A1:C1"), not a covered cell like "B1". Do not include the "⟨merged ...⟩" tag text in "quote". - For a continuous quote crossing two pages, set "page" to "N-M" and include [[PAGE_BREAK]] at the page break. Otherwise, use separate quote objects. - For legacy compatibility, you may also include top-level "page" and "quote" matching the first quote. +- TESTING ONLY: Make 50% of document citation quotes exact source text and 50% deliberately false text that does not appear in the source, so citation verification produces both outcomes. Use an even number of document quotes. Do not alter case-law citations. - Omit the block when there are no citations. DOCX GENERATION: diff --git a/backend/src/lib/chat/streaming.ts b/backend/src/lib/chat/streaming.ts index 41f175856..d718cb2be 100644 --- a/backend/src/lib/chat/streaming.ts +++ b/backend/src/lib/chat/streaming.ts @@ -532,8 +532,7 @@ export async function runLLMStream(params: { parseCitationsWithDiagnostics(fullText); let citations: unknown[]; if (buildCitations) { - // Custom builders (tabular) bypass verification; annotations carry no - // verification_status and the UI treats a missing status as untrusted. + // Custom builders (tabular) bypass document-citation verification. citations = buildCitations(fullText); } else { const rawCitations = parsedCitations.map((c) => diff --git a/backend/src/lib/chat/types.ts b/backend/src/lib/chat/types.ts index 12a87fb0d..42d3825d4 100644 --- a/backend/src/lib/chat/types.ts +++ b/backend/src/lib/chat/types.ts @@ -57,27 +57,13 @@ export type ChatMessage = { workflow?: { id: string; title: string }; }; -/** - * Result of server-side verification of a document quote against the extracted - * source text. - * - `verified` — the quote was found and is character-identical to the source. - * - `repaired` — the quote was found under whitespace/case/punctuation-tolerant - * matching but drifted from the source; `source_excerpt` holds the - * exact source text and has been swapped into the displayed quote. - * - `unverified` — no tolerant match; the model's quote is preserved but untrusted. - * - * A MISSING status (undefined) must be treated as untrusted by the UI — some - * paths (tabular, abort/error persistence) do not run verification. - */ -export type CitationVerificationStatus = "verified" | "unverified" | "repaired"; - /** * Per-quote verification result. `start_char`/`end_char` index into the * EXTRACTED source text (not the raw file bytes) and are only present for * single-segment quotes that matched. */ export type QuoteVerification = { - status: CitationVerificationStatus; + verified: boolean; start_char?: number; end_char?: number; source_excerpt?: string; diff --git a/backend/src/lib/chat/verifyCitations.test.ts b/backend/src/lib/chat/verifyCitations.test.ts index 78f285b26..5a6207341 100644 --- a/backend/src/lib/chat/verifyCitations.test.ts +++ b/backend/src/lib/chat/verifyCitations.test.ts @@ -51,7 +51,9 @@ describe("locateQuote", () => { }); it("returns null when the quote is absent", () => { - expect(locateQuote(SOURCE, "the Tenant shall vacate immediately")).toBeNull(); + expect( + locateQuote(SOURCE, "the Tenant shall vacate immediately"), + ).toBeNull(); }); }); @@ -59,82 +61,115 @@ describe("verifyQuoteAgainstSource", () => { it("exact match → verified with correct offsets into the source", () => { const quote = "The Landlord may terminate this Lease"; const v = verifyQuoteAgainstSource(SOURCE, quote); - expect(v.status).toBe("verified"); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(false); expect(v.source_excerpt).toBe(quote); expect(SOURCE.slice(v.start_char, v.end_char)).toBe(quote); }); - it("whitespace + case drift → repaired with the exact source excerpt", () => { + it("whitespace + case drift → verified with the exact source excerpt", () => { const drifted = "the landlord MAY terminate this lease"; const v = verifyQuoteAgainstSource(SOURCE, drifted); - expect(v.status).toBe("repaired"); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(true); expect(v.source_excerpt).toBe("The Landlord may terminate this Lease"); expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); }); - it("punctuation drift → repaired with corrected excerpt and offsets", () => { + it("punctuation drift → verified with corrected excerpt and offsets", () => { // Model inserted a stray comma the source does not contain. const drifted = "pay rent, on the first day"; const v = verifyQuoteAgainstSource(SOURCE, drifted); - expect(v.status).toBe("repaired"); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(true); expect(v.source_excerpt).toBe("pay rent on the first day"); expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); }); it("fabricated quote → unverified with no offsets", () => { const v = verifyQuoteAgainstSource(SOURCE, "The Tenant waives all rights."); - expect(v.status).toBe("unverified"); + expect(v.verified).toBe(false); expect(v.start_char).toBeUndefined(); expect(v.end_char).toBeUndefined(); expect(v.source_excerpt).toBeUndefined(); }); it("empty / unreadable source → unverified", () => { - expect(verifyQuoteAgainstSource("", "anything").status).toBe("unverified"); + expect(verifyQuoteAgainstSource("", "anything").verified).toBe(false); expect( - verifyQuoteAgainstSource("Document could not be read.", "anything").status, - ).toBe("unverified"); + verifyQuoteAgainstSource("Document could not be read.", "anything") + .verified, + ).toBe(false); }); it("cross-page [[PAGE_BREAK]] quote → each segment verified independently", () => { const src = "the first day of each month. The Landlord may terminate"; - const quote = "the first day of each month[[PAGE_BREAK]]The Landlord may terminate"; + const quote = + "the first day of each month[[PAGE_BREAK]]The Landlord may terminate"; const v = verifyQuoteAgainstSource(src, quote); - expect(v.status).toBe("verified"); + expect(v.verified).toBe(true); expect(v.source_excerpt).toContain("[[PAGE_BREAK]]"); }); it("cross-page quote with a missing segment → unverified", () => { const quote = "the first day of each month[[PAGE_BREAK]]never appears here"; const v = verifyQuoteAgainstSource(SOURCE, quote); - expect(v.status).toBe("unverified"); + expect(v.verified).toBe(false); + }); + + it.each(["...", "…"])( + "ellipsis-separated quote using %s verifies each excerpt independently", + (ellipsis) => { + const quote = `The Tenant shall pay rent${ellipsis}The Landlord may terminate this Lease`; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(true); + expect(v.source_excerpt).toBe( + "The Tenant shall pay rent ... The Landlord may terminate this Lease", + ); + }, + ); + + it("ellipsis-separated quote with a missing excerpt → unverified", () => { + const quote = + "The Tenant shall pay rent...the Tenant may sublet without consent"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(false); + }); + + it("supports ellipsis omissions inside a cross-page quote", () => { + const quote = + "The Tenant shall pay...first day of each month[[PAGE_BREAK]]The Landlord may terminate...written notice"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(true); + expect(v.source_excerpt).toContain("[[PAGE_BREAK]]"); }); }); describe("verifyDocumentCitationAnnotation", () => { const fetcher = fetcherFor({ "doc-1": SOURCE }); - it("marks a verified quote and attaches per-quote offsets + aggregate status", async () => { + it("marks a verified quote and attaches per-quote offsets", async () => { const ann = (await verifyDocumentCitationAnnotation( docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), fetcher, )) as Record; - expect(ann.verification_status).toBe("verified"); - const quotes = ann.quotes as { verification: { status: string } }[]; - expect(quotes[0].verification.status).toBe("verified"); + expect(ann.verified).toBe(true); + const quotes = ann.quotes as { verification: { verified: boolean } }[]; + expect(quotes[0].verification.verified).toBe(true); }); - it("repairs a drifted quote by swapping in the exact source excerpt", async () => { + it("corrects a drifted quote by swapping in the exact source excerpt", async () => { const ann = (await verifyDocumentCitationAnnotation( docAnnotation([{ page: 1, quote: "the TENANT shall pay rent" }]), fetcher, )) as Record; - expect(ann.verification_status).toBe("repaired"); + expect(ann.verified).toBe(true); const quotes = ann.quotes as { quote: string; - verification: { status: string; source_excerpt: string }; + verification: { verified: boolean; source_excerpt: string }; }[]; - expect(quotes[0].verification.status).toBe("repaired"); + expect(quotes[0].verification.verified).toBe(true); + expect(quotes[0].verification).not.toHaveProperty("needs_correction"); // The displayed quote is swapped to the true source text. expect(quotes[0].quote).toBe("The Tenant shall pay rent"); // Legacy top-level quote mirror is updated too. @@ -146,7 +181,7 @@ describe("verifyDocumentCitationAnnotation", () => { docAnnotation([{ page: 1, quote: "The Tenant may sublet freely." }]), fetcher, )) as Record; - expect(ann.verification_status).toBe("unverified"); + expect(ann.verified).toBe(false); const quotes = ann.quotes as { quote: string }[]; expect(quotes[0].quote).toBe("The Tenant may sublet freely."); }); @@ -159,7 +194,7 @@ describe("verifyDocumentCitationAnnotation", () => { ]), fetcher, )) as Record; - expect(ann.verification_status).toBe("unverified"); + expect(ann.verified).toBe(false); }); it("unreadable source → all quotes unverified", async () => { @@ -167,7 +202,7 @@ describe("verifyDocumentCitationAnnotation", () => { docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), fetcherFor({ "doc-1": "Document could not be read." }), )) as Record; - expect(ann.verification_status).toBe("unverified"); + expect(ann.verified).toBe(false); }); it("leaves case-law annotations untouched (no CourtListener regression)", async () => { @@ -177,26 +212,32 @@ describe("verifyDocumentCitationAnnotation", () => { ref: 2, cluster_id: 42, case_name: "Roe v. Doe", - quotes: [{ opinionId: null, type: null, author: null, quote: "held that…" }], + quotes: [ + { opinionId: null, type: null, author: null, quote: "held that…" }, + ], }; const out = await verifyDocumentCitationAnnotation(caseAnn, fetcher); expect(out).toBe(caseAnn); - expect(out as Record).not.toHaveProperty( - "verification_status", - ); + expect(out as Record).not.toHaveProperty("verified"); }); }); describe("verifyDocumentCitations (batch)", () => { it("verifies documents and passes case citations through unchanged", async () => { - const caseAnn = { type: "citation_data", kind: "case", ref: 2, cluster_id: 7 }; + const caseAnn = { + type: "citation_data", + kind: "case", + ref: 2, + cluster_id: 7, + }; const out = await verifyDocumentCitations( - [docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), caseAnn], + [ + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + caseAnn, + ], fetcherFor({ "doc-1": SOURCE }), ); - expect((out[0] as Record).verification_status).toBe( - "verified", - ); + expect((out[0] as Record).verified).toBe(true); expect(out[1]).toBe(caseAnn); }); }); diff --git a/backend/src/lib/chat/verifyCitations.ts b/backend/src/lib/chat/verifyCitations.ts index fb0d49afa..fd2ada77f 100644 --- a/backend/src/lib/chat/verifyCitations.ts +++ b/backend/src/lib/chat/verifyCitations.ts @@ -1,13 +1,11 @@ -import type { - CitationVerificationStatus, - QuoteVerification, -} from "./types"; +import type { QuoteVerification } from "./types"; import { normalizeWithMap } from "./tools/documentOps"; // Mirrors the frontend: cross-page quotes join two page segments with this // sentinel (see expandDocumentQuoteEntry in // frontend/src/app/components/shared/types.ts). const PAGE_BREAK_SENTINEL = "[[PAGE_BREAK]]"; +const ELLIPSIS_PATTERN = /\.{3}|…/; // Source-text sentinels returned by readDocumentContent when a document can't // be read. Treat these as "no source" so every quote falls back to unverified @@ -18,6 +16,9 @@ const UNREADABLE_SOURCES = new Set([ ]); type QuoteLocation = { start: number; end: number; excerpt: string }; +type QuoteVerificationResult = QuoteVerification & { + needs_correction: boolean; +}; /** * Locate `quote` inside `source`, returning the exact original substring @@ -28,7 +29,10 @@ type QuoteLocation = { start: number; end: number; excerpt: string }; * 3. whitespace + case + punctuation normalized (tolerant/fuzzy) * Offsets index into the EXTRACTED source text, not the raw file bytes. */ -export function locateQuote(source: string, quote: string): QuoteLocation | null { +export function locateQuote( + source: string, + quote: string, +): QuoteLocation | null { if (!source || !quote) return null; // Tier 1: exact. @@ -65,16 +69,16 @@ function locateNormalized( /** * Verify a single model quote against the source text, returning the - * per-quote verification record. Cross-page quotes (containing the - * `[[PAGE_BREAK]]` sentinel) are split and each segment verified independently; - * char offsets are only attached for single-segment quotes. + * per-quote verification record. Cross-page quotes and quotes abbreviated with + * `...` or `…` are split and each segment verified independently; char offsets + * are only attached for contiguous single-segment quotes. */ export function verifyQuoteAgainstSource( source: string, quote: string, -): QuoteVerification { +): QuoteVerificationResult { if (!source || UNREADABLE_SOURCES.has(source)) { - return { status: "unverified" }; + return { verified: false, needs_correction: false }; } if (quote.includes(PAGE_BREAK_SENTINEL)) { @@ -82,37 +86,63 @@ export function verifyQuoteAgainstSource( .split(PAGE_BREAK_SENTINEL) .map((s) => s.trim()) .filter((s) => s.length > 0); - if (!segments.length) return { status: "unverified" }; - const located = segments.map((seg) => ({ seg, loc: locateQuote(source, seg) })); - if (located.some((l) => !l.loc)) return { status: "unverified" }; - const anyRepaired = located.some((l) => l.loc!.excerpt !== l.seg); + if (!segments.length) return { verified: false, needs_correction: false }; + const verified = segments.map((seg) => + verifyQuoteAgainstSource(source, seg), + ); + if (verified.some((result) => !result.verified)) { + return { verified: false, needs_correction: false }; + } return { - status: anyRepaired ? "repaired" : "verified", - source_excerpt: located - .map((l) => l.loc!.excerpt) + verified: true, + needs_correction: verified.some((result) => result.needs_correction), + source_excerpt: verified + .map((result, index) => result.source_excerpt ?? segments[index]) .join(` ${PAGE_BREAK_SENTINEL} `), }; } + // The document viewers treat ASCII and Unicode ellipses as omission + // separators and highlight each quoted segment independently. Mirror that + // behavior here so a legitimate abbreviated quote is not rejected merely + // because text was intentionally omitted between its verbatim segments. + if (ELLIPSIS_PATTERN.test(quote)) { + const segments = quote + .split(ELLIPSIS_PATTERN) + .map((segment) => segment.trim()) + // Match the viewers' normalization: punctuation-only remnants (for + // example, the fourth dot in "....") do not form quoted segments. + .filter((segment) => /[\p{L}\p{N}]/u.test(segment)); + if (!segments.length) return { verified: false, needs_correction: false }; + const located = segments.map((segment) => ({ + segment, + location: locateQuote(source, segment), + })); + if (located.some(({ location }) => !location)) { + return { verified: false, needs_correction: false }; + } + return { + verified: true, + needs_correction: located.some( + ({ segment, location }) => location!.excerpt !== segment, + ), + source_excerpt: located + .map(({ location }) => location!.excerpt) + .join(" ... "), + }; + } + const loc = locateQuote(source, quote); - if (!loc) return { status: "unverified" }; + if (!loc) return { verified: false, needs_correction: false }; return { - status: loc.excerpt === quote ? "verified" : "repaired", + verified: true, + needs_correction: loc.excerpt !== quote, start_char: loc.start, end_char: loc.end, source_excerpt: loc.excerpt, }; } -/** Aggregate a list of per-quote statuses: unverified beats repaired beats verified. */ -function aggregateStatus( - statuses: CitationVerificationStatus[], -): CitationVerificationStatus { - if (statuses.some((s) => s === "unverified")) return "unverified"; - if (statuses.some((s) => s === "repaired")) return "repaired"; - return "verified"; -} - type DocQuoteEntry = { page: number | string; quote: string }; /** @@ -120,7 +150,7 @@ type DocQuoteEntry = { page: number | string; quote: string }; * Case-law annotations (kind === "case") are returned untouched — their * existence is verified upstream via CourtListener and must never be * re-marked. For document annotations, source text is fetched once via - * `getSourceText(doc_id)` and each quote is located in it; repaired quotes + * `getSourceText(doc_id)` and each quote is located in it; corrected quotes * have the exact source excerpt swapped in so the UI never shows drifted text. */ export async function verifyDocumentCitationAnnotation( @@ -148,25 +178,24 @@ export async function verifyDocumentCitationAnnotation( } const verifiedQuotes = entries.map((entry) => { - const verification = verifyQuoteAgainstSource(source, entry.quote); - // Swap the exact source text into the displayed quote when we repaired it, + const result = verifyQuoteAgainstSource(source, entry.quote); + const { needs_correction, ...verification } = result; + // Swap the exact source text into the displayed quote when it drifted, // so a drifted quote is never surfaced as the source's words. const quote = - verification.status === "repaired" && verification.source_excerpt + needs_correction && verification.source_excerpt ? verification.source_excerpt : entry.quote; return { ...entry, quote, verification }; }); - const verificationStatus = aggregateStatus( - verifiedQuotes.map((q) => q.verification.status), - ); + const verified = verifiedQuotes.every((q) => q.verification.verified); return { ...a, quote: verifiedQuotes[0]?.quote ?? a.quote, quotes: verifiedQuotes, - verification_status: verificationStatus, + verified, }; } diff --git a/frontend/src/app/components/assistant/CitationQuotesHeader.tsx b/frontend/src/app/components/assistant/CitationQuotesHeader.tsx index 1f5ceb970..ee1ab860e 100644 --- a/frontend/src/app/components/assistant/CitationQuotesHeader.tsx +++ b/frontend/src/app/components/assistant/CitationQuotesHeader.tsx @@ -3,6 +3,10 @@ import { useEffect, useState, type ReactNode } from "react"; import { Minus, RectangleHorizontal, Rows3 } from "lucide-react"; import { CiteButton } from "@/app/components/ui/cite-button"; +import { + CitationVerificationBadge, + type CitationVerificationDisplayState, +} from "./message/citationVerification"; export type CitationQuoteHeaderItem = { id: string; @@ -11,6 +15,7 @@ export type CitationQuoteHeaderItem = { inlineDetail?: string | null; detail?: string | null; citationText?: string | null; + verificationState?: CitationVerificationDisplayState; }; const QUOTE_GLASS_SURFACE = @@ -72,22 +77,37 @@ export function CitationQuotesHeader({ Quotes - {quotes.map((quote, index) => ( - - ))} + {quotes.map((quote, index) => { + const isUnverified = + quote.verificationState === + "unverified"; + return ( + + ); + })} )} + {currentQuote?.verificationState === "unverified" && ( + + )} {currentQuote && ( void; }) { + const isUnverified = quote.verificationState === "unverified"; + const isSelected = isActive && !isUnverified; + return (