diff --git a/backend/src/lib/chat/contextBuilders.test.ts b/backend/src/lib/chat/contextBuilders.test.ts new file mode 100644 index 000000000..8435c8327 --- /dev/null +++ b/backend/src/lib/chat/contextBuilders.test.ts @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { enrichWithPriorEvents } from "./contextBuilders"; + +test("prior-turn context identifies every legal-source provider attempt", async () => { + const events = [ + { + type: "legal_source_search", + provider_id: null, + provider_name: "Multiple legal sources", + query: "summary judgment", + result_count: 2, + providers: [ + { + provider_id: "a2aj-canada", + provider_name: "A2AJ", + status: "succeeded", + result_count: 2, + }, + { + provider_id: "ontario-elaws", + provider_name: "Ontario e-Laws", + status: "failed", + result_count: 0, + error_code: "http-503", + }, + ], + }, + ]; + const query = { + select() { + return this; + }, + eq() { + return this; + }, + order() { + return this; + }, + async limit() { + return { data: [{ content: events }] }; + }, + }; + const db = { + from() { + return query; + }, + }; + + const result = await enrichWithPriorEvents( + [ + { role: "assistant", content: "Earlier response." }, + { role: "user", content: "Which connectors were used?" }, + ], + "synthetic-chat", + db as never, + {}, + ); + const assistant = result[0].content ?? ""; + assert.match(assistant, /A2AJ: succeeded, 2 results/); + assert.match(assistant, /Ontario e-Laws: failed, 0 results, reason http-503/); +}); diff --git a/backend/src/lib/chat/contextBuilders.ts b/backend/src/lib/chat/contextBuilders.ts index c3ca5c034..96d8a0826 100644 --- a/backend/src/lib/chat/contextBuilders.ts +++ b/backend/src/lib/chat/contextBuilders.ts @@ -98,6 +98,54 @@ export async function enrichWithPriorEvents( ); } } + } else if (ev?.type === "legal_source_search") { + const attempts = Array.isArray(ev.providers) + ? (ev.providers as Record[]) + : []; + if (attempts.length > 0) { + for (const attempt of attempts) { + const name = + typeof attempt.provider_name === "string" + ? attempt.provider_name + : typeof attempt.provider_id === "string" + ? attempt.provider_id + : "unknown provider"; + const status = + attempt.status === "succeeded" ? "succeeded" : "failed"; + const count = + typeof attempt.result_count === "number" + ? `, ${attempt.result_count} result${attempt.result_count === 1 ? "" : "s"}` + : ""; + const code = + typeof attempt.error_code === "string" + ? `, reason ${attempt.error_code}` + : ""; + lines.push( + `- legal-source search → ${name}: ${status}${count}${code}`, + ); + } + } else { + const provider = + typeof ev.provider_name === "string" + ? ev.provider_name + : "legal sources"; + const count = + typeof ev.result_count === "number" ? ev.result_count : 0; + lines.push( + `- legal-source search → ${provider}: ${ev.error ? "failed" : "completed"}, ${count} result${count === 1 ? "" : "s"}`, + ); + } + } else if (ev?.type === "legal_authority") { + const provider = + typeof ev.provider_name === "string" + ? ev.provider_name + : typeof ev.provider_id === "string" + ? ev.provider_id + : "unknown provider"; + const action = typeof ev.action === "string" ? ev.action : "used"; + lines.push( + `- legal authority → ${provider}: ${ev.error ? `${action} failed` : action}`, + ); } } if (lines.length === 0) return messages; diff --git a/backend/src/lib/chat/legalResearchGate.test.ts b/backend/src/lib/chat/legalResearchGate.test.ts new file mode 100644 index 000000000..4bd22eb90 --- /dev/null +++ b/backend/src/lib/chat/legalResearchGate.test.ts @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { requiresLegalSourceSearch } from "./legalResearchGate"; + +const user = (content: string) => [{ role: "user" as const, content }]; + +test("requires source discovery for explicit legal research requests", () => { + assert.equal( + requiresLegalSourceSearch( + user("Research Ontario case law about summary judgment."), + ), + true, + ); + assert.equal( + requiresLegalSourceSearch( + user("Verify this citation and check the current statute."), + ), + true, + ); + assert.equal( + requiresLegalSourceSearch(user("What is the limitation law in Ontario?")), + true, + ); +}); + +test("does not start a new search for connector audit questions", () => { + assert.equal( + requiresLegalSourceSearch(user("Which connectors were actually used?")), + false, + ); + assert.equal( + requiresLegalSourceSearch(user("Please summarize the document I attached.")), + false, + ); + assert.equal( + requiresLegalSourceSearch( + user("Use CourtListener to search for a U.S. equal protection case."), + ), + false, + ); +}); diff --git a/backend/src/lib/chat/legalResearchGate.ts b/backend/src/lib/chat/legalResearchGate.ts new file mode 100644 index 000000000..865bdd050 --- /dev/null +++ b/backend/src/lib/chat/legalResearchGate.ts @@ -0,0 +1,39 @@ +import type { LlmMessage } from "../llm"; + +const CONNECTOR_AUDIT_QUESTION = + /\b(which|what)\s+(?:legal[- ]source\s+)?connectors?\s+(?:were|was|did|have)\b|\bconnectors?\s+(?:were|was)\s+(?:actually\s+)?used\b/i; + +const EXPLICIT_COURTLISTENER_REQUEST = /\bcourt\s*listener\b/i; + +const EXPLICIT_RESEARCH_REQUEST = + /\b(research|search|find|locate|look\s+up|verify|validate|check|cite|retrieve|not(?:e|ing)[ -]?up)\b/i; + +const LEGAL_AUTHORITY_SUBJECT = + /\b(case\s+law|cases?|decisions?|authorit(?:y|ies)|precedents?|citations?|statutes?|legislation|regulations?|rules?|practice\s+directions?|current\s+law|legal\s+sources?|canlii|a2aj|e-?laws?|justice\s+laws?)\b/i; + +const DIRECT_LEGAL_QUESTION = + /\b(what\s+is|what\s+are|does|do|can|when|whether|is|are)\b[\s\S]{0,120}\b(law|legal|statute|regulation|rule|court|appeal|limitation|jurisdiction)\b/i; + +/** + * The model remains free to use other tools after the first round, but a + * request that plainly asks for legal research must begin with source + * discovery. This prevents a polished model-memory answer from bypassing the + * authorized connector layer. + */ +export function requiresLegalSourceSearch(messages: LlmMessage[]): boolean { + const latestUser = [...messages] + .reverse() + .find((message) => message.role === "user") + ?.content.trim(); + if ( + !latestUser || + CONNECTOR_AUDIT_QUESTION.test(latestUser) || + EXPLICIT_COURTLISTENER_REQUEST.test(latestUser) + ) + return false; + return ( + (EXPLICIT_RESEARCH_REQUEST.test(latestUser) && + LEGAL_AUTHORITY_SUBJECT.test(latestUser)) || + DIRECT_LEGAL_QUESTION.test(latestUser) + ); +} diff --git a/backend/src/lib/chat/openaiToolLimit.test.ts b/backend/src/lib/chat/openaiToolLimit.test.ts index e604b8ea2..960eca95a 100644 --- a/backend/src/lib/chat/openaiToolLimit.test.ts +++ b/backend/src/lib/chat/openaiToolLimit.test.ts @@ -83,3 +83,61 @@ test("OpenAI tool limit performs a final synthesis turn", async () => { globalThis.fetch = originalFetch; } }); + +test("OpenAI can require legal-source discovery on only the first round", async () => { + const originalFetch = globalThis.fetch; + const requestBodies: Array> = []; + globalThis.fetch = async (_input, init) => { + requestBodies.push( + JSON.parse(String(init?.body)) as Record, + ); + return requestBodies.length === 1 + ? sse([ + { + type: "response.output_item.done", + item: { + type: "function_call", + call_id: "search-1", + name: "search_legal_sources", + arguments: + '{"query":"summary judgment","jurisdiction":"CA-ON","material_type":"decision"}', + }, + }, + ]) + : sse([ + { type: "response.output_text.delta", delta: "Researched answer." }, + ]); + }; + + try { + await streamOpenAI({ + model: "gpt-5.6", + systemPrompt: "Research before answering.", + messages: [{ role: "user", content: "Research summary judgment." }], + tools: [ + { + type: "function", + function: { + name: "search_legal_sources", + description: "Search authorized legal sources.", + parameters: { type: "object", properties: {} }, + }, + }, + ], + requiredFirstToolName: "search_legal_sources", + maxIterations: 1, + apiKeys: { openai: "sk-synthetic" }, + runTools: async () => [ + { tool_use_id: "search-1", content: '{"results":[]}' }, + ], + }); + + assert.deepEqual(requestBodies[0].tool_choice, { + type: "function", + name: "search_legal_sources", + }); + assert.equal(requestBodies[1].tool_choice, undefined); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/backend/src/lib/chat/streaming.ts b/backend/src/lib/chat/streaming.ts index 70d74178d..55db2ce2b 100644 --- a/backend/src/lib/chat/streaming.ts +++ b/backend/src/lib/chat/streaming.ts @@ -16,9 +16,11 @@ import { type CourtlistenerToolEvent, } from "./tools/courtlistenerTools"; import { + LEGAL_SOURCE_TOOL_NAMES, LEGAL_SOURCE_TOOLS, type LegalSourceToolEvent, } from "./tools/legalSourceTools"; +import { requiresLegalSourceSearch } from "./legalResearchGate"; import { type DocStore, type DocIndex, @@ -207,6 +209,10 @@ export async function runLLMStream(params: { role: m.role === "assistant" ? "assistant" : "user", content: m.content ?? "", })); + const requiredFirstToolName = + includeResearchTools && requiresLegalSourceSearch(chatMessages) + ? LEGAL_SOURCE_TOOL_NAMES.search + : undefined; const events: AssistantEvent[] = []; // One assistant turn produces at most one document_versions row per @@ -339,6 +345,7 @@ export async function runLLMStream(params: { systemPrompt, messages: chatMessages, tools: activeTools as OpenAIToolSchema[], + requiredFirstToolName, maxIterations: 10, apiKeys, enableThinking: true, diff --git a/backend/src/lib/chat/tools/legalSourceTools.ts b/backend/src/lib/chat/tools/legalSourceTools.ts index 699613b9f..29053768d 100644 --- a/backend/src/lib/chat/tools/legalSourceTools.ts +++ b/backend/src/lib/chat/tools/legalSourceTools.ts @@ -103,6 +103,14 @@ export type LegalSourceToolEvent = provider_name: string | null; query: string; result_count: number; + providers?: Array<{ + provider_id: string; + provider_name: string; + status: "succeeded" | "failed"; + result_count: number; + error_code?: string; + error?: string; + }>; coverage_warning?: string; error?: string; } diff --git a/backend/src/lib/chat/tools/toolDispatcher.ts b/backend/src/lib/chat/tools/toolDispatcher.ts index ae95b60fb..5bfd7fb41 100644 --- a/backend/src/lib/chat/tools/toolDispatcher.ts +++ b/backend/src/lib/chat/tools/toolDispatcher.ts @@ -493,6 +493,39 @@ function cleanLegalAuthority( }; } +function legalSourceFailure(error: unknown) { + const message = + error instanceof Error + ? error.message.slice(0, 500) + : "Legal source request failed."; + const status = Number((error as { status?: unknown })?.status); + const errorCode = + Number.isInteger(status) && status >= 400 + ? `http-${status}` + : error instanceof Error && error.name === "TimeoutError" + ? "timeout" + : /not configured|api key|credential|token/i.test(message) + ? "not-configured" + : /not enabled|disabled|entitlement|authorized/i.test(message) + ? "not-authorized" + : /invalid|unexpected|no legislation content/i.test(message) + ? "invalid-response" + : "provider-request-failed"; + const publicMessage = + errorCode === "timeout" + ? "The provider request timed out." + : errorCode === "not-configured" + ? "The provider is not configured for this user." + : errorCode === "not-authorized" + ? "The provider is not authorized for this request." + : errorCode === "invalid-response" + ? "The provider returned an invalid response." + : errorCode.startsWith("http-") + ? `The provider returned HTTP status ${errorCode.slice(5)}.` + : "The provider request failed."; + return { message: publicMessage, errorCode }; +} + async function executeLegalSourceTool(args: { name: string; input: Record; @@ -614,11 +647,14 @@ async function executeLegalSourceTool(args: { ) : []; return { provider: target.descriptor, results, available: true }; - } catch { + } catch (error) { + const failure = legalSourceFailure(error); return { provider: target.descriptor, results: [], available: false, + errorCode: failure.errorCode, + error: failure.message, }; } }), @@ -644,6 +680,13 @@ async function executeLegalSourceTool(args: { providers: searched.map((entry) => ({ ...entry.provider, available: entry.available, + result_count: entry.results.length, + ...(entry.available + ? {} + : { + error_code: entry.errorCode, + error: entry.error, + }), })), results, ...(coverageWarning ? { coverage_warning: coverageWarning } : {}), @@ -662,6 +705,18 @@ async function executeLegalSourceTool(args: { : "Multiple legal sources", query, result_count: results.length, + providers: searched.map((entry) => ({ + provider_id: entry.provider.id, + provider_name: entry.provider.name, + status: entry.available ? "succeeded" : "failed", + result_count: entry.results.length, + ...(entry.available + ? {} + : { + error_code: entry.errorCode, + error: entry.error, + }), + })), ...(coverageWarning ? { coverage_warning: coverageWarning } : {}), }, }; @@ -678,6 +733,8 @@ async function executeLegalSourceTool(args: { const results = await verifyCanadianCitations( parseCanadianCitations(text), providers, + (item) => + legalSourceProviderContext(item.descriptor.id, db, apiKeys), ); return { content: JSON.stringify({ @@ -784,10 +841,7 @@ async function executeLegalSourceTool(args: { }, }; } catch (error) { - const message = - error instanceof Error - ? error.message.slice(0, 500) - : "Legal source request failed."; + const { message } = legalSourceFailure(error); return { content: JSON.stringify({ error: message }), event: { diff --git a/backend/src/lib/legalSources/canadianCitations.test.ts b/backend/src/lib/legalSources/canadianCitations.test.ts index e1e6206ea..3999aba69 100644 --- a/backend/src/lib/legalSources/canadianCitations.test.ts +++ b/backend/src/lib/legalSources/canadianCitations.test.ts @@ -187,3 +187,38 @@ test("verification keeps citation, passage, currency, and treatment states separ assert.equal(statuteResult.passageVerification, "verified"); assert.equal(statuteResult.currencyVerification, "verified"); }); + +test("passes per-user provider context through citation verification", async () => { + let receivedToken: string | null | undefined; + const provider: LegalSourceProvider = { + descriptor: { + id: "canlii-licensed", + name: "CanLII metadata", + jurisdictions: ["CA-ON"], + kinds: ["decision"], + official: false, + fullTextStatus: "metadata-only", + enabledByDefault: false, + }, + health: async () => ({ ok: true }), + verifyCitations: async (_citations, context) => { + receivedToken = context?.apiToken; + return [ + { + input: "2024 ONCA 123", + providerId: "canlii-licensed", + status: "verified", + sourceId: "synthetic", + canonicalUrl: "https://www.canlii.org/", + }, + ]; + }, + }; + const [result] = await verifyCanadianCitations( + parseCanadianCitations("2024 ONCA 123"), + [provider], + () => ({ apiToken: "SYNTHETIC-USER-KEY" }), + ); + assert.equal(receivedToken, "SYNTHETIC-USER-KEY"); + assert.equal(result.citationVerification, "verified"); +}); diff --git a/backend/src/lib/legalSources/canadianCitations.ts b/backend/src/lib/legalSources/canadianCitations.ts index f74e318a6..4255891ce 100644 --- a/backend/src/lib/legalSources/canadianCitations.ts +++ b/backend/src/lib/legalSources/canadianCitations.ts @@ -1,5 +1,6 @@ import type { JurisdictionCode, + LegalSourceContext, LegalSourceProvider, VerificationState, } from "./types"; @@ -248,15 +249,23 @@ export function renderCanadianCitation( export async function verifyCanadianCitations( citations: ParsedCanadianCitation[], providers: LegalSourceProvider[], + contextForProvider: ( + provider: LegalSourceProvider, + ) => LegalSourceContext | undefined = () => undefined, ): Promise { return Promise.all( - citations.map((citation) => verifyOne(citation, providers)), + citations.map((citation) => + verifyOne(citation, providers, contextForProvider), + ), ); } async function verifyOne( citation: ParsedCanadianCitation, providers: LegalSourceProvider[], + contextForProvider: ( + provider: LegalSourceProvider, + ) => LegalSourceContext | undefined, ): Promise { const base: CanadianCitationVerification = { citation, @@ -280,7 +289,7 @@ async function verifyOne( const result = ( await provider.verifyCitations!([ stripPinpoint(citation.normalized), - ]) + ], contextForProvider(provider)) )[0]; if ( result?.status === "verified" || @@ -307,11 +316,14 @@ async function verifyOne( item.descriptor.jurisdictions.includes(citation.jurisdiction), )) { try { - const matches = await provider.searchLegislation!({ - query: stripPinpoint(citation.normalized), - jurisdiction: citation.jurisdiction, - limit: 10, - }); + const matches = await provider.searchLegislation!( + { + query: stripPinpoint(citation.normalized), + jurisdiction: citation.jurisdiction, + limit: 10, + }, + contextForProvider(provider), + ); const match = matches.find((item) => citationEquivalent(item.citation, citation.normalized), ); @@ -322,6 +334,7 @@ async function verifyOne( citation.pinpoint?.type === "rule" ? { section: citation.pinpoint.start } : undefined, + contextForProvider(provider), ); return { ...base, diff --git a/backend/src/lib/legalSources/officialLegislation.test.ts b/backend/src/lib/legalSources/officialLegislation.test.ts index c766766c6..ab5ff7b2c 100644 --- a/backend/src/lib/legalSources/officialLegislation.test.ts +++ b/backend/src/lib/legalSources/officialLegislation.test.ts @@ -122,3 +122,29 @@ test("official providers fail closed for inferred historical versions", async () /historical-version retrieval/, ); }); + +test("official provider health exercises the production retrieval path", async () => { + let ontarioRequests = 0; + const ontario = new OntarioELawsProvider(async (input) => { + ontarioRequests += 1; + const url = String(input); + return new Response( + url.endsWith("/currency-date") + ? "July 10, 2026" + : syntheticOntarioDocument, + { status: 200 }, + ); + }); + const justice = new JusticeLawsProvider(async () => + new Response(syntheticFederalXml, { status: 200 }), + ); + + assert.equal((await ontario.health()).ok, true); + assert.equal(ontarioRequests, 2); + assert.equal((await justice.health()).ok, true); + + const failed = new OntarioELawsProvider(async () => + new Response("unavailable", { status: 503 }), + ); + assert.equal((await failed.health()).ok, false); +}); diff --git a/backend/src/lib/legalSources/officialLegislation.ts b/backend/src/lib/legalSources/officialLegislation.ts index 72705e03a..6e463ceba 100644 --- a/backend/src/lib/legalSources/officialLegislation.ts +++ b/backend/src/lib/legalSources/officialLegislation.ts @@ -154,10 +154,31 @@ export class OntarioELawsProvider implements LegalSourceProvider { constructor(private readonly fetchImpl: typeof fetch = fetch) {} async health() { - return { - ok: true, - detail: "Official Ontario e-Laws links and permitted live retrieval are configured.", - }; + try { + const document = await this.fetchLegislation( + "ontario-statute-90c43", + { section: "1" }, + ); + return document.fullText && document.currentToDate + ? { + ok: true, + detail: + "Ontario e-Laws document and currency APIs returned current legislation.", + } + : { + ok: false, + detail: + "Ontario e-Laws retrieval did not return legislation text and currency metadata.", + }; + } catch (error) { + return { + ok: false, + detail: + error instanceof Error + ? error.message.slice(0, 300) + : "Ontario e-Laws retrieval failed.", + }; + } } async searchLegislation(input: { @@ -243,10 +264,30 @@ export class JusticeLawsProvider implements LegalSourceProvider { constructor(private readonly fetchImpl: typeof fetch = fetch) {} async health() { - return { - ok: true, - detail: "Department of Justice XML repository and official stable links are configured.", - }; + try { + const document = await this.fetchLegislation("federal-act-d-3.4", { + section: "1", + }); + return document.fullText && document.currentToDate + ? { + ok: true, + detail: + "Justice Laws production XML path returned current legislation.", + } + : { + ok: false, + detail: + "Justice Laws retrieval did not return legislation text and currency metadata.", + }; + } catch (error) { + return { + ok: false, + detail: + error instanceof Error + ? error.message.slice(0, 300) + : "Justice Laws retrieval failed.", + }; + } } async searchLegislation(input: { diff --git a/backend/src/lib/llm/claude.ts b/backend/src/lib/llm/claude.ts index b455d984c..18c0f624d 100644 --- a/backend/src/lib/llm/claude.ts +++ b/backend/src/lib/llm/claude.ts @@ -137,6 +137,17 @@ export async function streamClaude( tools: toolsEnabled && claudeTools.length ? (claudeTools as unknown as Tool[]) : undefined, + ...(iter === 0 && + toolsEnabled && + params.requiredFirstToolName + ? { + tool_choice: { + type: "tool", + name: params.requiredFirstToolName, + disable_parallel_tool_use: true, + }, + } + : {}), max_tokens: MAX_TOKENS, // Claude 4.x models require `thinking.type: "adaptive"` and // drive effort via `output_config.effort` rather than a fixed diff --git a/backend/src/lib/llm/gemini.ts b/backend/src/lib/llm/gemini.ts index 9f67f79fb..266378a64 100644 --- a/backend/src/lib/llm/gemini.ts +++ b/backend/src/lib/llm/gemini.ts @@ -1,4 +1,4 @@ -import { GoogleGenAI } from "@google/genai"; +import { FunctionCallingConfigMode, GoogleGenAI } from "@google/genai"; import type { StreamChatParams, StreamChatResult, @@ -196,6 +196,18 @@ export async function streamGemini( tools: toolsEnabled && functionDeclarations.length ? [{ functionDeclarations } as never] : undefined, + ...(iter === 0 && + toolsEnabled && + params.requiredFirstToolName + ? { + toolConfig: { + functionCallingConfig: { + mode: FunctionCallingConfigMode.ANY, + allowedFunctionNames: [params.requiredFirstToolName], + }, + }, + } + : {}), // When enabled, ask Gemini to surface thought summaries. // When disabled, explicitly zero the thinking budget so the // model skips thinking entirely (saves tokens and latency diff --git a/backend/src/lib/llm/openai.ts b/backend/src/lib/llm/openai.ts index e1728849f..b86c23036 100644 --- a/backend/src/lib/llm/openai.ts +++ b/backend/src/lib/llm/openai.ts @@ -172,6 +172,7 @@ async function createResponse(params: { previousResponseId?: string; reasoningSummary?: boolean; reasoningEffort?: ReasoningEffort; + requiredToolName?: string; apiKey: string; signal?: AbortSignal; }): Promise { @@ -186,6 +187,10 @@ async function createResponse(params: { instructions: params.instructions || undefined, input: params.input, tools: params.tools?.length ? params.tools : undefined, + tool_choice: + params.tools?.length && params.requiredToolName + ? { type: "function", name: params.requiredToolName } + : undefined, stream: params.stream, max_output_tokens: params.maxTokens ?? MAX_OUTPUT_TOKENS, previous_response_id: params.previousResponseId, @@ -256,6 +261,10 @@ export async function streamOpenAI( previousResponseId, reasoningSummary: !!enableThinking, reasoningEffort, + requiredToolName: + iter === 0 && toolsEnabled + ? params.requiredFirstToolName + : undefined, apiKey: key, signal: params.abortSignal, }); diff --git a/backend/src/lib/llm/openaiCompatible.ts b/backend/src/lib/llm/openaiCompatible.ts index 70f100390..d32a1ef1b 100644 --- a/backend/src/lib/llm/openaiCompatible.ts +++ b/backend/src/lib/llm/openaiCompatible.ts @@ -100,6 +100,7 @@ async function createChatCompletion(params: { model: string; messages: ChatMessage[]; tools?: OpenAIToolSchema[]; + requiredToolName?: string; stream: boolean; maxTokens?: number; apiKey: string; @@ -116,7 +117,14 @@ async function createChatCompletion(params: { model: params.model, messages: params.messages, tools: params.tools?.length ? normalizedTools(params.tools) : undefined, - tool_choice: params.tools?.length ? "auto" : undefined, + tool_choice: params.tools?.length + ? params.requiredToolName + ? { + type: "function", + function: { name: params.requiredToolName }, + } + : "auto" + : undefined, stream: params.stream, max_tokens: params.maxTokens ?? 16_384, }), @@ -150,6 +158,10 @@ export async function streamOpenAICompatible( model: params.model, messages, tools: toolsEnabled ? params.tools : [], + requiredToolName: + iteration === 0 && toolsEnabled + ? params.requiredFirstToolName + : undefined, stream: true, apiKey: key, signal: params.abortSignal, diff --git a/backend/src/lib/llm/types.ts b/backend/src/lib/llm/types.ts index 860535a8d..82afc4c06 100644 --- a/backend/src/lib/llm/types.ts +++ b/backend/src/lib/llm/types.ts @@ -60,6 +60,12 @@ export type StreamChatParams = { systemPrompt: string; messages: LlmMessage[]; tools?: OpenAIToolSchema[]; + /** + * Require this function on the first tool-enabled model round. Later rounds + * return to automatic tool selection so the model can fetch, find passages, + * and synthesize normally. + */ + requiredFirstToolName?: string; maxIterations?: number; callbacks?: StreamCallbacks; runTools?: (calls: NormalizedToolCall[]) => Promise; diff --git a/frontend/src/app/components/assistant/AssistantMessage.tsx b/frontend/src/app/components/assistant/AssistantMessage.tsx index dc903c1bd..d879a30f7 100644 --- a/frontend/src/app/components/assistant/AssistantMessage.tsx +++ b/frontend/src/app/components/assistant/AssistantMessage.tsx @@ -512,6 +512,18 @@ export function AssistantMessage({ > Searched legal sources{" "} {detail} + {event.providers && event.providers.length > 1 && ( +
    + {event.providers.map((provider) => ( +
  • + {provider.provider_name}:{" "} + {provider.status === "succeeded" + ? `${provider.result_count} ${provider.result_count === 1 ? "result" : "results"}` + : `failed${provider.error_code ? ` (${provider.error_code})` : ""}`} +
    • + ))} +
+ )} {event.coverage_warning && (

{event.coverage_warning} diff --git a/frontend/src/app/components/shared/types.ts b/frontend/src/app/components/shared/types.ts index 2ff9b2066..c29add321 100644 --- a/frontend/src/app/components/shared/types.ts +++ b/frontend/src/app/components/shared/types.ts @@ -265,6 +265,14 @@ export type AssistantEvent = provider_name: string | null; query: string; result_count: number; + providers?: Array<{ + provider_id: string; + provider_name: string; + status: "succeeded" | "failed"; + result_count: number; + error_code?: string; + error?: string; + }>; coverage_warning?: string; error?: string; isStreaming?: boolean; diff --git a/frontend/src/app/hooks/useAssistantChat.ts b/frontend/src/app/hooks/useAssistantChat.ts index b6893220e..f601dbe85 100644 --- a/frontend/src/app/hooks/useAssistantChat.ts +++ b/frontend/src/app/hooks/useAssistantChat.ts @@ -649,6 +649,12 @@ export function useAssistantChat({ query: typeof data.query === "string" ? data.query : "", result_count: typeof data.result_count === "number" ? data.result_count : 0, + providers: Array.isArray(data.providers) + ? (data.providers as Extract< + AssistantEvent, + { type: "legal_source_search" } + >["providers"]) + : undefined, coverage_warning: typeof data.coverage_warning === "string" ? data.coverage_warning diff --git a/reports/release-manifest-v1.json b/reports/release-manifest-v1.json index f9921c397..165db21f4 100644 --- a/reports/release-manifest-v1.json +++ b/reports/release-manifest-v1.json @@ -127,8 +127,8 @@ }, { "path": "backend/src/lib/llm/types.ts", - "sha256": "a585e8529fcc375be24e558fd5dbce4653a157d8ba43ed31c76cb168cb42cace", - "sizeBytes": 2056 + "sha256": "d4f0f4b57456bb44ac77d6b45d7bde61ac6036a98d551baf4ad62fbc1a633ceb", + "sizeBytes": 2290 }, { "path": "backend/src/lib/quarantinedUpload.ts", @@ -152,8 +152,8 @@ }, { "path": "backend/src/lib/chat/tools/toolDispatcher.ts", - "sha256": "b8e1cdc5f8d083582299d81a82080f8c5c1ea48fdc751dbca12595ef980982a8", - "sizeBytes": 78933 + "sha256": "c0e2560cbc2907b88d308ec20c22f28950708394ec229dcbadd02ac379493174", + "sizeBytes": 81126 }, { "path": "backend/src/middleware/dataBoundary.test.ts", @@ -502,8 +502,8 @@ }, { "path": "scripts/lib/live-source-observer.mjs", - "sha256": "602d07e0d1a4db162e8760e940efdfa338c7869f7324427d125b741203dc1601", - "sizeBytes": 6751 + "sha256": "fd5b45219305cacbba1bbab1fc056ec38d712d71636c8b85105caa0286609c9c", + "sizeBytes": 10713 }, { "path": "scripts/lib/release-train.mjs", diff --git a/scripts/lib/live-source-observer.mjs b/scripts/lib/live-source-observer.mjs index 71ec522be..6e97605b6 100644 --- a/scripts/lib/live-source-observer.mjs +++ b/scripts/lib/live-source-observer.mjs @@ -6,8 +6,8 @@ const REQUIRED_TARGETS = [ }, { id: "ontario-elaws", - url: "https://www.ontario.ca/laws/statute/90c43", - kind: "nonempty-text", + url: "https://www.ontario.ca/laws/api/v2/legislation/en", + kind: "ontario-runtime", }, { id: "justice-laws-canada", @@ -32,6 +32,18 @@ const a2ajRows = (payload) => { return []; }; +const a2ajSearchRows = (payload) => + Array.isArray(payload?.results) + ? payload.results + : Array.isArray(payload) + ? payload + : []; + +const a2ajCitation = (row) => + [row?.citation_en, row?.citation_fr, row?.citation2_en, row?.citation2_fr] + .find((value) => typeof value === "string" && value.trim()) + ?.trim() ?? null; + const reasonCode = (error) => { if (error?.name === "TimeoutError") return "timeout"; const status = Number(error?.status); @@ -46,6 +58,8 @@ const requestHeaders = (target) => ({ Accept: target.kind === "a2aj-split-coverage" ? "application/json" + : target.kind === "ontario-runtime" + ? "application/json, text/plain" : "text/html, application/xml, text/xml", "User-Agent": "ROSS-RanadeOSS-source-observer/1.0", }); @@ -128,11 +142,123 @@ async function inspectA2ajCoverage(fetchImpl, target, timeoutMs) { const versions = groups .map((group) => responseVersion(group.response, null)) .filter(Boolean); + const apiBase = target.url.replace(/\/coverage$/, ""); + for (const probe of [ + { docType: "cases", dataset: "ONCA", query: "law" }, + { + docType: "laws", + dataset: "LEGISLATION-ON", + query: "Act", + }, + ]) { + const searchParams = new URLSearchParams({ + query: probe.query, + doc_type: probe.docType, + dataset: probe.dataset, + size: "1", + }); + const searchResponse = await fetchResponse( + fetchImpl, + `${apiBase}/search?${searchParams}`, + target, + timeoutMs, + ); + if (!searchResponse.ok) { + const error = new Error("A2AJ production search returned an error."); + error.status = searchResponse.status; + throw error; + } + const result = a2ajSearchRows(await searchResponse.json())[0]; + const citation = a2ajCitation(result); + if (!citation) { + const error = new Error("A2AJ production search returned no citation."); + error.code = "invalid-payload"; + throw error; + } + const fetchParams = new URLSearchParams({ + citation, + doc_type: probe.docType, + }); + const documentResponse = await fetchResponse( + fetchImpl, + `${apiBase}/fetch?${fetchParams}`, + target, + timeoutMs, + ); + if (!documentResponse.ok) { + const error = new Error("A2AJ production fetch returned an error."); + error.status = documentResponse.status; + throw error; + } + const document = await documentResponse.json(); + const row = + document?.result ?? + (Array.isArray(document?.results) ? document.results[0] : document); + const text = [row?.unofficial_text_en, row?.unofficial_text_fr].find( + (value) => typeof value === "string" && value.trim(), + ); + if (!text) { + const error = new Error("A2AJ production fetch returned no source text."); + error.code = "invalid-payload"; + throw error; + } + } return { sourceVersion: versions.length === groups.length ? versions.join("|") - : `coverage-${cases.rows.length}-cases-${laws.rows.length}-laws`, + : `coverage-${cases.rows.length}-cases-${laws.rows.length}-laws-search-fetch-ok`, + }; +} + +async function inspectOntarioRuntime(fetchImpl, target, timeoutMs) { + const [documentResponse, currencyResponse] = await Promise.all([ + fetchResponse( + fetchImpl, + `${target.url}/doc-search/statute/90c43`, + target, + timeoutMs, + ), + fetchResponse( + fetchImpl, + `${target.url}/currency-date`, + target, + timeoutMs, + ), + ]); + if (!documentResponse.ok || !currencyResponse.ok) { + const error = new Error("Ontario e-Laws production API returned an error."); + error.status = !documentResponse.ok + ? documentResponse.status + : currencyResponse.status; + throw error; + } + const documentBody = await documentResponse.text(); + const currencyBody = await currencyResponse.text(); + let documentPayload; + try { + documentPayload = JSON.parse(documentBody); + } catch { + const error = new Error("Ontario e-Laws document response was invalid."); + error.code = "invalid-payload"; + throw error; + } + if ( + typeof documentPayload?.content !== "string" || + documentPayload.content.trim().length < 500 || + !/\b(?:18|19|20)\d{2}\b/.test(currencyBody) + ) { + const error = new Error( + "Ontario e-Laws production APIs returned incomplete content.", + ); + error.code = "invalid-payload"; + throw error; + } + return { + sourceVersion: [ + responseVersion(documentResponse, `document-${documentBody.length}`), + responseVersion(currencyResponse, currencyBody.trim().slice(0, 40)), + ].join("|"), }; } @@ -151,6 +277,8 @@ export async function observeLiveLegalSources({ const observation = target.kind === "a2aj-split-coverage" ? await inspectA2ajCoverage(fetchImpl, target, timeoutMs) + : target.kind === "ontario-runtime" + ? await inspectOntarioRuntime(fetchImpl, target, timeoutMs) : await (async () => { const response = await fetchResponse( fetchImpl, diff --git a/tests/operations/live-source-observer.test.mjs b/tests/operations/live-source-observer.test.mjs index 04e0852be..616706c85 100644 --- a/tests/operations/live-source-observer.test.mjs +++ b/tests/operations/live-source-observer.test.mjs @@ -18,11 +18,35 @@ const lawCoverage = JSON.stringify({ ], }); const legislation = `${"Ontario and Canadian law ".repeat(30)}`; +const ontarioDocument = JSON.stringify({ + content: `

${"Ontario legislation content ".repeat(30)}

`, +}); const requestedA2ajDocTypes = []; const successfulFetch = async (url) => { if (url.includes("api.a2aj.ca")) { - const docType = new URL(url).searchParams.get("doc_type"); + const parsed = new URL(url); + const docType = parsed.searchParams.get("doc_type"); + if (parsed.pathname === "/search") { + return Response.json({ + results: [ + { + citation_en: + docType === "laws" + ? "R.S.O. 1990, c. C.43" + : "2024 ONCA 123", + }, + ], + }); + } + if (parsed.pathname === "/fetch") { + return Response.json({ + result: { + citation_en: parsed.searchParams.get("citation"), + unofficial_text_en: "SYNTHETIC source passage.", + }, + }); + } requestedA2ajDocTypes.push(docType); return new Response(docType === "laws" ? lawCoverage : caseCoverage, { status: 200, @@ -32,6 +56,12 @@ const successfulFetch = async (url) => { }, }); } + if (url.includes("ontario.ca/laws/api")) { + return new Response( + url.endsWith("/currency-date") ? "July 10, 2026" : ontarioDocument, + { status: 200 }, + ); + } return new Response(legislation, { status: 200 }); }; @@ -58,7 +88,7 @@ test("live source observation records only sanitized operational metadata", asyn test("a required provider failure degrades the observation without exposing response bodies", async () => { const fetchImpl = async (url) => { if (!url.includes("api.a2aj.ca")) - return new Response(legislation, { status: 200 }); + return successfulFetch(url); const docType = new URL(url).searchParams.get("doc_type"); return docType === "laws" ? new Response("private upstream diagnostic", { status: 503 }) @@ -77,7 +107,7 @@ test("a required provider failure degrades the observation without exposing resp test("split coverage validation rejects an Ontario law missing from the laws response", async () => { const fetchImpl = async (url) => { if (!url.includes("api.a2aj.ca")) - return new Response(legislation, { status: 200 }); + return successfulFetch(url); const docType = new URL(url).searchParams.get("doc_type"); return new Response( docType === "laws"