Harden Canadian legal connector reliability

backend/src/lib/chat/contextBuilders.test.ts

@@ -0,0 +1,62 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { enrichWithPriorEvents } from "./contextBuilders";
+
+test("prior-turn context identifies every legal-source provider attempt", async () => {
+  const events = [
+    {
+      type: "legal_source_search",
+      provider_id: null,
+      provider_name: "Multiple legal sources",
+      query: "summary judgment",
+      result_count: 2,
+      providers: [
+        {
+          provider_id: "a2aj-canada",
+          provider_name: "A2AJ",
+          status: "succeeded",
+          result_count: 2,
+        },
+        {
+          provider_id: "ontario-elaws",
+          provider_name: "Ontario e-Laws",
+          status: "failed",
+          result_count: 0,
+          error_code: "http-503",
+        },
+      ],
+    },
+  ];
+  const query = {
+    select() {
+      return this;
+    },
+    eq() {
+      return this;
+    },
+    order() {
+      return this;
+    },
+    async limit() {
+      return { data: [{ content: events }] };
+    },
+  };
+  const db = {
+    from() {
+      return query;
+    },
+  };
+
+  const result = await enrichWithPriorEvents(
+    [
+      { role: "assistant", content: "Earlier response." },
+      { role: "user", content: "Which connectors were used?" },
+    ],
+    "synthetic-chat",
+    db as never,
+    {},
+  );
+  const assistant = result[0].content ?? "";
+  assert.match(assistant, /A2AJ: succeeded, 2 results/);
+  assert.match(assistant, /Ontario e-Laws: failed, 0 results, reason http-503/);
+});

backend/src/lib/chat/contextBuilders.ts

@@ -98,6 +98,54 @@ export async function enrichWithPriorEvents(
           );
         }
       }
+    } else if (ev?.type === "legal_source_search") {
+      const attempts = Array.isArray(ev.providers)
+        ? (ev.providers as Record<string, unknown>[])
+        : [];
+      if (attempts.length > 0) {
+        for (const attempt of attempts) {
+          const name =
+            typeof attempt.provider_name === "string"
+              ? attempt.provider_name
+              : typeof attempt.provider_id === "string"
+                ? attempt.provider_id
+                : "unknown provider";
+          const status =
+            attempt.status === "succeeded" ? "succeeded" : "failed";
+          const count =
+            typeof attempt.result_count === "number"
+              ? `, ${attempt.result_count} result${attempt.result_count === 1 ? "" : "s"}`
+              : "";
+          const code =
+            typeof attempt.error_code === "string"
+              ? `, reason ${attempt.error_code}`
+              : "";
+          lines.push(
+            `- legal-source search → ${name}: ${status}${count}${code}`,
+          );
+        }
+      } else {
+        const provider =
+          typeof ev.provider_name === "string"
+            ? ev.provider_name
+            : "legal sources";
+        const count =
+          typeof ev.result_count === "number" ? ev.result_count : 0;
+        lines.push(
+          `- legal-source search → ${provider}: ${ev.error ? "failed" : "completed"}, ${count} result${count === 1 ? "" : "s"}`,
+        );
+      }
+    } else if (ev?.type === "legal_authority") {
+      const provider =
+        typeof ev.provider_name === "string"
+          ? ev.provider_name
+          : typeof ev.provider_id === "string"
+            ? ev.provider_id
+            : "unknown provider";
+      const action = typeof ev.action === "string" ? ev.action : "used";
+      lines.push(
+        `- legal authority → ${provider}: ${ev.error ? `${action} failed` : action}`,
+      );
     }
   }
   if (lines.length === 0) return messages;

backend/src/lib/chat/legalResearchGate.test.ts

@@ -0,0 +1,41 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { requiresLegalSourceSearch } from "./legalResearchGate";
+
+const user = (content: string) => [{ role: "user" as const, content }];
+
+test("requires source discovery for explicit legal research requests", () => {
+  assert.equal(
+    requiresLegalSourceSearch(
+      user("Research Ontario case law about summary judgment."),
+    ),
+    true,
+  );
+  assert.equal(
+    requiresLegalSourceSearch(
+      user("Verify this citation and check the current statute."),
+    ),
+    true,
+  );
+  assert.equal(
+    requiresLegalSourceSearch(user("What is the limitation law in Ontario?")),
+    true,
+  );
+});
+
+test("does not start a new search for connector audit questions", () => {
+  assert.equal(
+    requiresLegalSourceSearch(user("Which connectors were actually used?")),
+    false,
+  );
+  assert.equal(
+    requiresLegalSourceSearch(user("Please summarize the document I attached.")),
+    false,
+  );
+  assert.equal(
+    requiresLegalSourceSearch(
+      user("Use CourtListener to search for a U.S. equal protection case."),
+    ),
+    false,
+  );
+});

backend/src/lib/chat/legalResearchGate.ts

@@ -0,0 +1,39 @@
+import type { LlmMessage } from "../llm";
+
+const CONNECTOR_AUDIT_QUESTION =
+  /\b(which|what)\s+(?:legal[- ]source\s+)?connectors?\s+(?:were|was|did|have)\b|\bconnectors?\s+(?:were|was)\s+(?:actually\s+)?used\b/i;
+
+const EXPLICIT_COURTLISTENER_REQUEST = /\bcourt\s*listener\b/i;
+
+const EXPLICIT_RESEARCH_REQUEST =
+  /\b(research|search|find|locate|look\s+up|verify|validate|check|cite|retrieve|not(?:e|ing)[ -]?up)\b/i;
+
+const LEGAL_AUTHORITY_SUBJECT =
+  /\b(case\s+law|cases?|decisions?|authorit(?:y|ies)|precedents?|citations?|statutes?|legislation|regulations?|rules?|practice\s+directions?|current\s+law|legal\s+sources?|canlii|a2aj|e-?laws?|justice\s+laws?)\b/i;
+
+const DIRECT_LEGAL_QUESTION =
+  /\b(what\s+is|what\s+are|does|do|can|when|whether|is|are)\b[\s\S]{0,120}\b(law|legal|statute|regulation|rule|court|appeal|limitation|jurisdiction)\b/i;
+
+/**
+ * The model remains free to use other tools after the first round, but a
+ * request that plainly asks for legal research must begin with source
+ * discovery. This prevents a polished model-memory answer from bypassing the
+ * authorized connector layer.
+ */
+export function requiresLegalSourceSearch(messages: LlmMessage[]): boolean {
+  const latestUser = [...messages]
+    .reverse()
+    .find((message) => message.role === "user")
+    ?.content.trim();
+  if (
+    !latestUser ||
+    CONNECTOR_AUDIT_QUESTION.test(latestUser) ||
+    EXPLICIT_COURTLISTENER_REQUEST.test(latestUser)
+  )
+    return false;
+  return (
+    (EXPLICIT_RESEARCH_REQUEST.test(latestUser) &&
+      LEGAL_AUTHORITY_SUBJECT.test(latestUser)) ||
+    DIRECT_LEGAL_QUESTION.test(latestUser)
+  );
+}

backend/src/lib/chat/openaiToolLimit.test.ts

@@ -83,3 +83,61 @@ test("OpenAI tool limit performs a final synthesis turn", async () => {
     globalThis.fetch = originalFetch;
   }
 });
+
+test("OpenAI can require legal-source discovery on only the first round", async () => {
+  const originalFetch = globalThis.fetch;
+  const requestBodies: Array<Record<string, unknown>> = [];
+  globalThis.fetch = async (_input, init) => {
+    requestBodies.push(
+      JSON.parse(String(init?.body)) as Record<string, unknown>,
+    );
+    return requestBodies.length === 1
+      ? sse([
+          {
+            type: "response.output_item.done",
+            item: {
+              type: "function_call",
+              call_id: "search-1",
+              name: "search_legal_sources",
+              arguments:
+                '{"query":"summary judgment","jurisdiction":"CA-ON","material_type":"decision"}',
+            },
+          },
+        ])
+      : sse([
+          { type: "response.output_text.delta", delta: "Researched answer." },
+        ]);
+  };
+
+  try {
+    await streamOpenAI({
+      model: "gpt-5.6",
+      systemPrompt: "Research before answering.",
+      messages: [{ role: "user", content: "Research summary judgment." }],
+      tools: [
+        {
+          type: "function",
+          function: {
+            name: "search_legal_sources",
+            description: "Search authorized legal sources.",
+            parameters: { type: "object", properties: {} },
+          },
+        },
+      ],
+      requiredFirstToolName: "search_legal_sources",
+      maxIterations: 1,
+      apiKeys: { openai: "sk-synthetic" },
+      runTools: async () => [
+        { tool_use_id: "search-1", content: '{"results":[]}' },
+      ],
+    });
+
+    assert.deepEqual(requestBodies[0].tool_choice, {
+      type: "function",
+      name: "search_legal_sources",
+    });
+    assert.equal(requestBodies[1].tool_choice, undefined);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});

backend/src/lib/chat/streaming.ts

@@ -16,9 +16,11 @@ import {
   type CourtlistenerToolEvent,
 } from "./tools/courtlistenerTools";
 import {
+  LEGAL_SOURCE_TOOL_NAMES,
   LEGAL_SOURCE_TOOLS,
   type LegalSourceToolEvent,
 } from "./tools/legalSourceTools";
+import { requiresLegalSourceSearch } from "./legalResearchGate";
 import {
   type DocStore,
   type DocIndex,
@@ -207,6 +209,10 @@ export async function runLLMStream(params: {
       role: m.role === "assistant" ? "assistant" : "user",
       content: m.content ?? "",
     }));
+  const requiredFirstToolName =
+    includeResearchTools && requiresLegalSourceSearch(chatMessages)
+      ? LEGAL_SOURCE_TOOL_NAMES.search
+      : undefined;
 
   const events: AssistantEvent[] = [];
   // One assistant turn produces at most one document_versions row per
@@ -339,6 +345,7 @@ export async function runLLMStream(params: {
       systemPrompt,
       messages: chatMessages,
       tools: activeTools as OpenAIToolSchema[],
+      requiredFirstToolName,
       maxIterations: 10,
       apiKeys,
       enableThinking: true,

backend/src/lib/chat/tools/legalSourceTools.ts

@@ -103,6 +103,14 @@ export type LegalSourceToolEvent =
       provider_name: string | null;
       query: string;
       result_count: number;
+      providers?: Array<{
+        provider_id: string;
+        provider_name: string;
+        status: "succeeded" | "failed";
+        result_count: number;
+        error_code?: string;
+        error?: string;
+      }>;
       coverage_warning?: string;
       error?: string;
     }