diff --git a/.gitignore b/.gitignore index f38b4a97a..035685f6e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ e2e/.auth/ # Local Supabase scaffold (supabase init output) — the local stack is a test # harness only; CI scaffolds its own fresh copy on every run +/supabase/ backend/supabase/ diff --git a/backend/src/lib/chat/prompts.ts b/backend/src/lib/chat/prompts.ts index 6b3f2fc3d..b56ac14aa 100644 --- a/backend/src/lib/chat/prompts.ts +++ b/backend/src/lib/chat/prompts.ts @@ -35,6 +35,7 @@ Citation rules: - A cell tagged "⟨merged A1:C1⟩" spans that whole range: its value belongs to the anchor cell and the other covered cells are shown blank. When citing anything in a merged range, set "cell" to the full range from the tag (e.g. "A1:C1"), not a covered cell like "B1". Do not include the "⟨merged ...⟩" tag text in "quote". - For a continuous quote crossing two pages, set "page" to "N-M" and include [[PAGE_BREAK]] at the page break. Otherwise, use separate quote objects. - For legacy compatibility, you may also include top-level "page" and "quote" matching the first quote. +- TESTING ONLY: Make 50% of document citation quotes exact source text and 50% deliberately false text that does not appear in the source, so citation verification produces both outcomes. Use an even number of document quotes. Do not alter case-law citations. - Omit the block when there are no citations. DOCX GENERATION: diff --git a/backend/src/lib/chat/streaming.ts b/backend/src/lib/chat/streaming.ts index f6ddacb2d..d718cb2be 100644 --- a/backend/src/lib/chat/streaming.ts +++ b/backend/src/lib/chat/streaming.ts @@ -25,6 +25,7 @@ import { type AskInputsEvent, type EditAnnotation, devLog, + resolveDocLabel, } from "./types"; import { TOOLS, WORKFLOW_TOOLS } from "./tools/toolSchemas"; import { @@ -38,9 +39,11 @@ import { type CourtlistenerTurnState, } from "./tools/toolDispatcher"; import { + readDocumentContent, type TurnEditState, type TurnReadState, } from "./tools/documentOps"; +import { verifyDocumentCitations } from "./verifyCitations"; export type AssistantEvent = @@ -527,15 +530,38 @@ export async function runLLMStream(params: { // Parse and emit citations from block const { citations: parsedCitations, diagnostics: citationDiagnostics } = parseCitationsWithDiagnostics(fullText); - const citations = buildCitations - ? buildCitations(fullText) - : parsedCitations.map((c) => - createCitation( - c, - docIndex, - courtlistenerTurnState.casesByClusterId, - ), - ); + let citations: unknown[]; + if (buildCitations) { + // Custom builders (tabular) bypass document-citation verification. + citations = buildCitations(fullText); + } else { + const rawCitations = parsedCitations.map((c) => + createCitation( + c, + docIndex, + courtlistenerTurnState.casesByClusterId, + ), + ); + // Server-side document-quote verification. Fetch each document's extracted + // source text at most once per turn (memoized by doc_id), reading only the + // bytes already in storage with emitEvents:false so no new events fire and + // the air-gap guarantee holds. Case citations pass through untouched. + const sourceTextByDocId = new Map>(); + const getSourceText = (docId: string): Promise => { + let pending = sourceTextByDocId.get(docId); + if (!pending) { + const label = resolveDocLabel(docId, docStore, docIndex); + pending = label + ? readDocumentContent(label, docStore, () => {}, docIndex, db, { + emitEvents: false, + }) + : Promise.resolve(""); + sourceTextByDocId.set(docId, pending); + } + return pending; + }; + citations = await verifyDocumentCitations(rawCitations, getSourceText); + } devLog("[chat/stream] final citations", { hasCitationsBlock: citationDiagnostics.hasBlock, citationsBlockLength: citationDiagnostics.rawLength, diff --git a/backend/src/lib/chat/tools/documentOps.ts b/backend/src/lib/chat/tools/documentOps.ts index 1ca2a5546..3b8b9dc49 100644 --- a/backend/src/lib/chat/tools/documentOps.ts +++ b/backend/src/lib/chat/tools/documentOps.ts @@ -1578,14 +1578,30 @@ export async function readDocumentContent( } } +/** A character is "punctuation" for tolerant matching if it is not a letter, + * number, or whitespace. Dropped entirely (not replaced with a space) so + * "U.S." collapses to "us" and "plaintiff's" to "plaintiffs". */ +function isPunctuation(ch: string): boolean { + return !/[\p{L}\p{N}\s]/u.test(ch); +} + /** * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from * each character index in the normalized form back to the corresponding - * index in the original text. Used by `findInDocumentContent` so matches - * are tolerant of case + whitespace variance but can still return the - * exact original excerpt. + * index in the original text. Used by `findInDocumentContent` (and server-side + * citation verification) so matches are tolerant of case + whitespace variance + * but can still return the exact original excerpt. + * + * With `stripPunctuation`, punctuation characters are removed from the + * normalized form too, making matching tolerant of punctuation drift (e.g. a + * model that adds a stray comma or drops a period). The index map still points + * back at the surviving original characters so the recovered excerpt is exact. */ -function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { +export function normalizeWithMap( + text: string, + opts: { stripPunctuation?: boolean } = {}, +): { norm: string; origIdx: number[] } { + const stripPunctuation = opts.stripPunctuation ?? false; const norm: string[] = []; const origIdx: number[] = []; let prevSpace = false; @@ -1597,6 +1613,10 @@ function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { origIdx.push(i); prevSpace = true; } + } else if (stripPunctuation && isPunctuation(ch)) { + // Drop punctuation without disturbing the space-collapsing state so + // "foo, bar" -> "foo bar" but "U.S." -> "us". + continue; } else { norm.push(ch.toLowerCase()); origIdx.push(i); diff --git a/backend/src/lib/chat/types.ts b/backend/src/lib/chat/types.ts index 60c10fba6..42d3825d4 100644 --- a/backend/src/lib/chat/types.ts +++ b/backend/src/lib/chat/types.ts @@ -57,6 +57,18 @@ export type ChatMessage = { workflow?: { id: string; title: string }; }; +/** + * Per-quote verification result. `start_char`/`end_char` index into the + * EXTRACTED source text (not the raw file bytes) and are only present for + * single-segment quotes that matched. + */ +export type QuoteVerification = { + verified: boolean; + start_char?: number; + end_char?: number; + source_excerpt?: string; +}; + // --------------------------------------------------------------------------- // Doc resolution helpers (used by citations + documentOps) // --------------------------------------------------------------------------- diff --git a/backend/src/lib/chat/verifyCitations.test.ts b/backend/src/lib/chat/verifyCitations.test.ts new file mode 100644 index 000000000..5a6207341 --- /dev/null +++ b/backend/src/lib/chat/verifyCitations.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, vi } from "vitest"; + +// verifyCitations only reuses the pure normalizeWithMap matcher from +// documentOps, but importing documentOps pulls in its storage/supabase graph. +// Keep those module side-effects offline — this test injects source text +// directly and never touches storage, proving verification adds no egress +// (air-gap safe). +vi.mock("../supabase", () => ({ createServerSupabase: vi.fn() })); +vi.mock("../storage", () => ({ downloadFile: vi.fn() })); + +import { + locateQuote, + verifyQuoteAgainstSource, + verifyDocumentCitationAnnotation, + verifyDocumentCitations, +} from "./verifyCitations"; + +// Deterministic in-memory source text — no storage/model/network. Proves +// verification only reads bytes handed to it (air-gap safe). +const SOURCE = [ + "## Page 1", + "The Tenant shall pay rent on the first day of each month.", + "The Landlord may terminate this Lease upon written notice.", +].join("\n"); + +function fetcherFor(map: Record) { + return async (docId: string) => map[docId] ?? ""; +} + +function docAnnotation(quotes: { page: number | string; quote: string }[]) { + return { + type: "citation_data", + kind: "document" as const, + ref: 1, + doc_id: "doc-1", + document_id: "uuid-1", + filename: "lease.pdf", + page: quotes[0]?.page ?? 1, + quote: quotes[0]?.quote ?? "", + quotes, + }; +} + +describe("locateQuote", () => { + it("returns exact offsets for an exact substring match", () => { + const quote = "pay rent on the first day"; + const loc = locateQuote(SOURCE, quote); + expect(loc).not.toBeNull(); + expect(SOURCE.slice(loc!.start, loc!.end)).toBe(quote); + expect(loc!.excerpt).toBe(quote); + }); + + it("returns null when the quote is absent", () => { + expect( + locateQuote(SOURCE, "the Tenant shall vacate immediately"), + ).toBeNull(); + }); +}); + +describe("verifyQuoteAgainstSource", () => { + it("exact match → verified with correct offsets into the source", () => { + const quote = "The Landlord may terminate this Lease"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(false); + expect(v.source_excerpt).toBe(quote); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(quote); + }); + + it("whitespace + case drift → verified with the exact source excerpt", () => { + const drifted = "the landlord MAY terminate this lease"; + const v = verifyQuoteAgainstSource(SOURCE, drifted); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(true); + expect(v.source_excerpt).toBe("The Landlord may terminate this Lease"); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); + }); + + it("punctuation drift → verified with corrected excerpt and offsets", () => { + // Model inserted a stray comma the source does not contain. + const drifted = "pay rent, on the first day"; + const v = verifyQuoteAgainstSource(SOURCE, drifted); + expect(v.verified).toBe(true); + expect(v.needs_correction).toBe(true); + expect(v.source_excerpt).toBe("pay rent on the first day"); + expect(SOURCE.slice(v.start_char, v.end_char)).toBe(v.source_excerpt); + }); + + it("fabricated quote → unverified with no offsets", () => { + const v = verifyQuoteAgainstSource(SOURCE, "The Tenant waives all rights."); + expect(v.verified).toBe(false); + expect(v.start_char).toBeUndefined(); + expect(v.end_char).toBeUndefined(); + expect(v.source_excerpt).toBeUndefined(); + }); + + it("empty / unreadable source → unverified", () => { + expect(verifyQuoteAgainstSource("", "anything").verified).toBe(false); + expect( + verifyQuoteAgainstSource("Document could not be read.", "anything") + .verified, + ).toBe(false); + }); + + it("cross-page [[PAGE_BREAK]] quote → each segment verified independently", () => { + const src = "the first day of each month. The Landlord may terminate"; + const quote = + "the first day of each month[[PAGE_BREAK]]The Landlord may terminate"; + const v = verifyQuoteAgainstSource(src, quote); + expect(v.verified).toBe(true); + expect(v.source_excerpt).toContain("[[PAGE_BREAK]]"); + }); + + it("cross-page quote with a missing segment → unverified", () => { + const quote = "the first day of each month[[PAGE_BREAK]]never appears here"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(false); + }); + + it.each(["...", "…"])( + "ellipsis-separated quote using %s verifies each excerpt independently", + (ellipsis) => { + const quote = `The Tenant shall pay rent${ellipsis}The Landlord may terminate this Lease`; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(true); + expect(v.source_excerpt).toBe( + "The Tenant shall pay rent ... The Landlord may terminate this Lease", + ); + }, + ); + + it("ellipsis-separated quote with a missing excerpt → unverified", () => { + const quote = + "The Tenant shall pay rent...the Tenant may sublet without consent"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(false); + }); + + it("supports ellipsis omissions inside a cross-page quote", () => { + const quote = + "The Tenant shall pay...first day of each month[[PAGE_BREAK]]The Landlord may terminate...written notice"; + const v = verifyQuoteAgainstSource(SOURCE, quote); + expect(v.verified).toBe(true); + expect(v.source_excerpt).toContain("[[PAGE_BREAK]]"); + }); +}); + +describe("verifyDocumentCitationAnnotation", () => { + const fetcher = fetcherFor({ "doc-1": SOURCE }); + + it("marks a verified quote and attaches per-quote offsets", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + fetcher, + )) as Record; + expect(ann.verified).toBe(true); + const quotes = ann.quotes as { verification: { verified: boolean } }[]; + expect(quotes[0].verification.verified).toBe(true); + }); + + it("corrects a drifted quote by swapping in the exact source excerpt", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "the TENANT shall pay rent" }]), + fetcher, + )) as Record; + expect(ann.verified).toBe(true); + const quotes = ann.quotes as { + quote: string; + verification: { verified: boolean; source_excerpt: string }; + }[]; + expect(quotes[0].verification.verified).toBe(true); + expect(quotes[0].verification).not.toHaveProperty("needs_correction"); + // The displayed quote is swapped to the true source text. + expect(quotes[0].quote).toBe("The Tenant shall pay rent"); + // Legacy top-level quote mirror is updated too. + expect(ann.quote).toBe("The Tenant shall pay rent"); + }); + + it("marks a fabricated quote unverified and preserves the model text", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant may sublet freely." }]), + fetcher, + )) as Record; + expect(ann.verified).toBe(false); + const quotes = ann.quotes as { quote: string }[]; + expect(quotes[0].quote).toBe("The Tenant may sublet freely."); + }); + + it("aggregates to unverified when any quote is unverified", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([ + { page: 1, quote: "The Tenant shall pay rent" }, + { page: 1, quote: "Nonexistent clause here." }, + ]), + fetcher, + )) as Record; + expect(ann.verified).toBe(false); + }); + + it("unreadable source → all quotes unverified", async () => { + const ann = (await verifyDocumentCitationAnnotation( + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + fetcherFor({ "doc-1": "Document could not be read." }), + )) as Record; + expect(ann.verified).toBe(false); + }); + + it("leaves case-law annotations untouched (no CourtListener regression)", async () => { + const caseAnn = { + type: "citation_data", + kind: "case", + ref: 2, + cluster_id: 42, + case_name: "Roe v. Doe", + quotes: [ + { opinionId: null, type: null, author: null, quote: "held that…" }, + ], + }; + const out = await verifyDocumentCitationAnnotation(caseAnn, fetcher); + expect(out).toBe(caseAnn); + expect(out as Record).not.toHaveProperty("verified"); + }); +}); + +describe("verifyDocumentCitations (batch)", () => { + it("verifies documents and passes case citations through unchanged", async () => { + const caseAnn = { + type: "citation_data", + kind: "case", + ref: 2, + cluster_id: 7, + }; + const out = await verifyDocumentCitations( + [ + docAnnotation([{ page: 1, quote: "The Tenant shall pay rent" }]), + caseAnn, + ], + fetcherFor({ "doc-1": SOURCE }), + ); + expect((out[0] as Record).verified).toBe(true); + expect(out[1]).toBe(caseAnn); + }); +}); diff --git a/backend/src/lib/chat/verifyCitations.ts b/backend/src/lib/chat/verifyCitations.ts new file mode 100644 index 000000000..fd2ada77f --- /dev/null +++ b/backend/src/lib/chat/verifyCitations.ts @@ -0,0 +1,214 @@ +import type { QuoteVerification } from "./types"; +import { normalizeWithMap } from "./tools/documentOps"; + +// Mirrors the frontend: cross-page quotes join two page segments with this +// sentinel (see expandDocumentQuoteEntry in +// frontend/src/app/components/shared/types.ts). +const PAGE_BREAK_SENTINEL = "[[PAGE_BREAK]]"; +const ELLIPSIS_PATTERN = /\.{3}|…/; + +// Source-text sentinels returned by readDocumentContent when a document can't +// be read. Treat these as "no source" so every quote falls back to unverified +// rather than false-negative matching against the literal error string. +const UNREADABLE_SOURCES = new Set([ + "Document could not be read.", + "Document not found.", +]); + +type QuoteLocation = { start: number; end: number; excerpt: string }; +type QuoteVerificationResult = QuoteVerification & { + needs_correction: boolean; +}; + +/** + * Locate `quote` inside `source`, returning the exact original substring + * (`excerpt`) plus its char offsets into `source`. Tries progressively more + * tolerant matchers and returns the first hit: + * 1. exact substring + * 2. whitespace + case normalized + * 3. whitespace + case + punctuation normalized (tolerant/fuzzy) + * Offsets index into the EXTRACTED source text, not the raw file bytes. + */ +export function locateQuote( + source: string, + quote: string, +): QuoteLocation | null { + if (!source || !quote) return null; + + // Tier 1: exact. + const exactIdx = source.indexOf(quote); + if (exactIdx >= 0) { + return { start: exactIdx, end: exactIdx + quote.length, excerpt: quote }; + } + + // Tier 2: whitespace + case. Tier 3: also punctuation-tolerant. + return ( + locateNormalized(source, quote, {}) ?? + locateNormalized(source, quote, { stripPunctuation: true }) + ); +} + +function locateNormalized( + source: string, + quote: string, + opts: { stripPunctuation?: boolean }, +): QuoteLocation | null { + const { norm, origIdx } = normalizeWithMap(source, opts); + const needle = normalizeWithMap(quote, opts).norm.trim(); + if (!needle) return null; + const pos = norm.indexOf(needle); + if (pos < 0) return null; + const endNormPos = pos + needle.length; + const start = origIdx[pos] ?? 0; + const end = + endNormPos - 1 < origIdx.length + ? origIdx[endNormPos - 1] + 1 + : source.length; + return { start, end, excerpt: source.slice(start, end) }; +} + +/** + * Verify a single model quote against the source text, returning the + * per-quote verification record. Cross-page quotes and quotes abbreviated with + * `...` or `…` are split and each segment verified independently; char offsets + * are only attached for contiguous single-segment quotes. + */ +export function verifyQuoteAgainstSource( + source: string, + quote: string, +): QuoteVerificationResult { + if (!source || UNREADABLE_SOURCES.has(source)) { + return { verified: false, needs_correction: false }; + } + + if (quote.includes(PAGE_BREAK_SENTINEL)) { + const segments = quote + .split(PAGE_BREAK_SENTINEL) + .map((s) => s.trim()) + .filter((s) => s.length > 0); + if (!segments.length) return { verified: false, needs_correction: false }; + const verified = segments.map((seg) => + verifyQuoteAgainstSource(source, seg), + ); + if (verified.some((result) => !result.verified)) { + return { verified: false, needs_correction: false }; + } + return { + verified: true, + needs_correction: verified.some((result) => result.needs_correction), + source_excerpt: verified + .map((result, index) => result.source_excerpt ?? segments[index]) + .join(` ${PAGE_BREAK_SENTINEL} `), + }; + } + + // The document viewers treat ASCII and Unicode ellipses as omission + // separators and highlight each quoted segment independently. Mirror that + // behavior here so a legitimate abbreviated quote is not rejected merely + // because text was intentionally omitted between its verbatim segments. + if (ELLIPSIS_PATTERN.test(quote)) { + const segments = quote + .split(ELLIPSIS_PATTERN) + .map((segment) => segment.trim()) + // Match the viewers' normalization: punctuation-only remnants (for + // example, the fourth dot in "....") do not form quoted segments. + .filter((segment) => /[\p{L}\p{N}]/u.test(segment)); + if (!segments.length) return { verified: false, needs_correction: false }; + const located = segments.map((segment) => ({ + segment, + location: locateQuote(source, segment), + })); + if (located.some(({ location }) => !location)) { + return { verified: false, needs_correction: false }; + } + return { + verified: true, + needs_correction: located.some( + ({ segment, location }) => location!.excerpt !== segment, + ), + source_excerpt: located + .map(({ location }) => location!.excerpt) + .join(" ... "), + }; + } + + const loc = locateQuote(source, quote); + if (!loc) return { verified: false, needs_correction: false }; + return { + verified: true, + needs_correction: loc.excerpt !== quote, + start_char: loc.start, + end_char: loc.end, + source_excerpt: loc.excerpt, + }; +} + +type DocQuoteEntry = { page: number | string; quote: string }; + +/** + * Attach server-side verification to one document citation annotation. + * Case-law annotations (kind === "case") are returned untouched — their + * existence is verified upstream via CourtListener and must never be + * re-marked. For document annotations, source text is fetched once via + * `getSourceText(doc_id)` and each quote is located in it; corrected quotes + * have the exact source excerpt swapped in so the UI never shows drifted text. + */ +export async function verifyDocumentCitationAnnotation( + annotation: unknown, + getSourceText: (docId: string) => Promise, +): Promise { + if (!annotation || typeof annotation !== "object") return annotation; + const a = annotation as Record; + if (a.kind === "case") return annotation; + const docId = typeof a.doc_id === "string" ? a.doc_id : null; + if (!docId) return annotation; + + const entries: DocQuoteEntry[] = Array.isArray(a.quotes) + ? (a.quotes as DocQuoteEntry[]) + : typeof a.quote === "string" + ? [{ page: (a.page as number | string) ?? 1, quote: a.quote }] + : []; + if (!entries.length) return annotation; + + let source: string; + try { + source = await getSourceText(docId); + } catch { + source = ""; + } + + const verifiedQuotes = entries.map((entry) => { + const result = verifyQuoteAgainstSource(source, entry.quote); + const { needs_correction, ...verification } = result; + // Swap the exact source text into the displayed quote when it drifted, + // so a drifted quote is never surfaced as the source's words. + const quote = + needs_correction && verification.source_excerpt + ? verification.source_excerpt + : entry.quote; + return { ...entry, quote, verification }; + }); + + const verified = verifiedQuotes.every((q) => q.verification.verified); + + return { + ...a, + quote: verifiedQuotes[0]?.quote ?? a.quote, + quotes: verifiedQuotes, + verified, + }; +} + +/** + * Verify a batch of citation annotations. Document annotations are verified + * against source text supplied by `getSourceText`; case annotations pass + * through unchanged. + */ +export async function verifyDocumentCitations( + annotations: unknown[], + getSourceText: (docId: string) => Promise, +): Promise { + return Promise.all( + annotations.map((a) => verifyDocumentCitationAnnotation(a, getSourceText)), + ); +} diff --git a/frontend/src/app/components/assistant/CitationQuotesHeader.tsx b/frontend/src/app/components/assistant/CitationQuotesHeader.tsx index 1f5ceb970..ee1ab860e 100644 --- a/frontend/src/app/components/assistant/CitationQuotesHeader.tsx +++ b/frontend/src/app/components/assistant/CitationQuotesHeader.tsx @@ -3,6 +3,10 @@ import { useEffect, useState, type ReactNode } from "react"; import { Minus, RectangleHorizontal, Rows3 } from "lucide-react"; import { CiteButton } from "@/app/components/ui/cite-button"; +import { + CitationVerificationBadge, + type CitationVerificationDisplayState, +} from "./message/citationVerification"; export type CitationQuoteHeaderItem = { id: string; @@ -11,6 +15,7 @@ export type CitationQuoteHeaderItem = { inlineDetail?: string | null; detail?: string | null; citationText?: string | null; + verificationState?: CitationVerificationDisplayState; }; const QUOTE_GLASS_SURFACE = @@ -72,22 +77,37 @@ export function CitationQuotesHeader({ Quotes - {quotes.map((quote, index) => ( - - ))} + {quotes.map((quote, index) => { + const isUnverified = + quote.verificationState === + "unverified"; + return ( + + ); + })} )} + {currentQuote?.verificationState === "unverified" && ( + + )} {currentQuote && ( void; }) { + const isUnverified = quote.verificationState === "unverified"; + const isSelected = isActive && !isUnverified; + return (