Fix provider-specific legal-source release gates

.github/workflows/verify-ontario-sources.yml

@@ -26,12 +26,14 @@ jobs:
           package-manager-cache: false
 
       - name: Run sanitized live observations
+        id: legal-source-observation
         run: node scripts/observe-legal-sources.mjs --output artifacts/legal-source-health-live.json
 
       - name: Upload sanitized observation
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: legal-source-health-${{ github.run_id }}
           path: artifacts/legal-source-health-live.json
-          if-no-files-found: error
+          if-no-files-found: warn
           retention-days: 14

reports/release-manifest-v1.json

@@ -57,8 +57,8 @@
     },
     {
       "path": ".github/workflows/verify-ontario-sources.yml",
-      "sha256": "a686858060c8c90be6860e57d7f97bbc4a3d00c79c72c337bc471b4f7cd88e03",
-      "sizeBytes": 916
+      "sha256": "f2155aefc27585c8e1d81e7fc5dd749b44538cbbf14da0674ee0328daf534b65",
+      "sizeBytes": 973
     },
     {
       "path": "backend/migrations/20260718_01_document_scan_pipeline.sql",
@@ -502,8 +502,8 @@
     },
     {
       "path": "scripts/lib/live-source-observer.mjs",
-      "sha256": "5af7622207e75ca1a7c8102da0ad8017008c17bbeed09bedcdc94510ae73bd5b",
-      "sizeBytes": 13801
+      "sha256": "2d8032e837ba2f0ba36a974184bdd5230bc077e50d1fdc349fa53cd2eb323de5",
+      "sizeBytes": 16112
     },
     {
       "path": "scripts/lib/release-train.mjs",
@@ -532,13 +532,13 @@
     },
     {
       "path": "scripts/fly-release-train.mjs",
-      "sha256": "71338c98da92f12dfc244c3eb49d74b00cde169e5b8cdfbe5597ec75954199d0",
-      "sizeBytes": 26940
+      "sha256": "b50581d83d645b0f05399090dd7016ddfd6cabf85d00b2b4210fe2885274c306",
+      "sizeBytes": 28844
     },
     {
       "path": "scripts/observe-legal-sources.mjs",
-      "sha256": "11a1c2e066ebaa55d4a001a7f3148183e5179838bb88e40deec2e6f45a7bc6e4",
-      "sizeBytes": 1082
+      "sha256": "ecbdcb7f83550bdfee2fafebb494d263de165ee2d7a6fb2a3f21cc886e849b58",
+      "sizeBytes": 1351
     },
     {
       "path": "scripts/preflight-fly-images.sh",
@@ -607,8 +607,8 @@
     },
     {
       "path": "tests/baseline/ross-delivery-d.test.mjs",
-      "sha256": "e6e0763eb8afdfa2e1fd262e1024b0463eae8a36e5de11c187e3c1b119bb1b83",
-      "sizeBytes": 2742
+      "sha256": "6a8dab18934f81a206d58c85cefdaa87a344a94519ef3282e65afbaa10b7dcc9",
+      "sizeBytes": 2789
     },
     {
       "path": "tests/baseline/ross-production-readiness.test.mjs",
@@ -617,8 +617,8 @@
     },
     {
       "path": "tests/baseline/ross-release-train.test.mjs",
-      "sha256": "2406c9046010025257a3bd9586f7821010a8e1b35717a66f7c4fbd12ed712720",
-      "sizeBytes": 27604
+      "sha256": "5b64b90d4496365264852feeba59556a1b35350488755dcb8d2fec4ad12379a1",
+      "sizeBytes": 29357
     },
     {
       "path": "tests/baseline/ross-staging-debug.test.mjs",

scripts/fly-release-train.mjs

@@ -6,6 +6,7 @@ import {
     existsSync,
     mkdirSync,
     readFileSync,
+    unlinkSync,
     writeFileSync,
 } from "node:fs";
 import { resolve } from "node:path";
@@ -402,6 +403,66 @@ function runRemoteProbe(app, machineId, args) {
     );
 }
 
+function observeLegalSources(outputPath) {
+    const resolvedOutputPath = resolve(root, outputPath);
+    if (existsSync(resolvedOutputPath)) unlinkSync(resolvedOutputPath);
+
+    const result = run(
+        "node",
+        [
+            "scripts/observe-legal-sources.mjs",
+            "--output",
+            outputPath,
+        ],
+        { capture: true, allowFailure: true },
+    );
+    if (result.stdout) process.stdout.write(result.stdout);
+    if (result.stderr) process.stderr.write(result.stderr);
+
+    if (!existsSync(resolvedOutputPath)) {
+        throw new Error(
+            `Live legal-source observer did not produce a sanitized report at ${outputPath}.`,
+        );
+    }
+
+    let report;
+    try {
+        report = JSON.parse(readFileSync(resolvedOutputPath, "utf8"));
+    } catch (error) {
+        throw new Error(
+            `Live legal-source observer produced an unreadable sanitized report: ${error instanceof Error ? error.message : String(error)}.`,
+        );
+    }
+
+    const providerStates = Object.fromEntries(
+        Object.entries(report.providers ?? {}).map(([id, item]) => [
+            id,
+            {
+                state: item?.state ?? "unknown",
+                reasonCode: item?.reasonCode ?? "unknown",
+                attempts: item?.attempts ?? null,
+            },
+        ]),
+    );
+    ledger.rehearsal.legalSourceObservation = report.status ?? "unknown";
+    ledger.rehearsal.legalSourceProviders = providerStates;
+    ledger.rehearsal.legalSourceObserverExitCode = result.status;
+    saveLedger();
+
+    if (result.status !== 0 || report.status !== "healthy") {
+        const summary = Object.entries(providerStates)
+            .map(
+                ([id, item]) =>
+                    `${id}=${item.state} (${item.reasonCode}, attempts=${item.attempts ?? "unknown"})`,
+            )
+            .join(", ");
+        throw new Error(
+            `Live legal-source observation ${report.status ?? "unknown"}; provider states: ${summary}.`,
+        );
+    }
+    return report;
+}
+
 function wakePublicService(url) {
     run("curl", [
         "--fail",
@@ -688,14 +749,11 @@ function rehearse() {
             });
             verifySet(stageApps, candidate);
             smoke("rehearsal", { full: true });
-            run("node", [
-                "scripts/observe-legal-sources.mjs",
-                "--output",
+            observeLegalSources(
                 "artifacts/release-train-legal-source-health.json",
-            ]);
+            );
             ledger.rehearsal.candidatePromotionVerified = true;
             ledger.rehearsal.readOnlyIntegrationChecks = "passed";
-            ledger.rehearsal.legalSourceObservation = "healthy";
         } catch (error) {
             ledger.rehearsal.candidateFailureRollbackAttempted = true;
             try {

scripts/lib/live-source-observer.mjs

@@ -1,21 +1,39 @@
-const REQUIRED_TARGETS = [
+const LIVE_TARGETS = [
   {
     id: "a2aj-canada",
     url: "https://api.a2aj.ca/coverage",
     kind: "a2aj-split-coverage",
+    required: false,
   },
   {
     id: "ontario-elaws",
     url: "https://www.ontario.ca/laws/api/v2/legislation/en",
     kind: "ontario-runtime",
+    required: true,
   },
   {
     id: "justice-laws-canada",
     url: "https://raw.githubusercontent.com/justicecanada/laws-lois-xml/main",
     kind: "justice-runtime",
+    required: true,
   },
 ];
 
+const RETRYABLE_HTTP_STATUSES = new Set([
+  408,
+  425,
+  429,
+  500,
+  502,
+  503,
+  504,
+]);
+const DEFAULT_RETRY_ATTEMPTS = 3;
+const DEFAULT_RETRY_DELAY_MS = 250;
+
+const wait = (milliseconds) =>
+  new Promise((resolve) => setTimeout(resolve, milliseconds));
+
 const latencyClass = (milliseconds) =>
   milliseconds < 1_000 ? "fast" : milliseconds < 5_000 ? "standard" : "slow";
 
@@ -74,6 +92,7 @@ async function fetchResponse(fetchImpl, url, target, timeoutMs) {
 
 async function inspect(response, target) {
   if (!response.ok) {
+    await response.body?.cancel();
     const error = new Error("Legal source returned a non-success status.");
     error.status = response.status;
     throw error;
@@ -96,6 +115,39 @@ async function inspect(response, target) {
   return `content-${body.length}-bytes`;
 }
 
+function isRetryableError(target, error) {
+  const status = Number(error?.status);
+  return (
+    error?.name === "TimeoutError" ||
+    RETRYABLE_HTTP_STATUSES.has(status) ||
+    (target.id === "ontario-elaws" && error?.code === "invalid-payload")
+  );
+}
+
+async function observeWithRetries(
+  target,
+  operation,
+  { retryAttempts, retryDelayMs, sleepImpl },
+) {
+  let lastError = null;
+  let attempts = 0;
+  for (let attempt = 1; attempt <= retryAttempts; attempt += 1) {
+    attempts = attempt;
+    try {
+      return {
+        ok: true,
+        attempts: attempt,
+        observation: await operation(),
+      };
+    } catch (error) {
+      lastError = error;
+      if (attempt === retryAttempts || !isRetryableError(target, error)) break;
+      await sleepImpl(retryDelayMs * 2 ** (attempt - 1));
+    }
+  }
+  return { ok: false, attempts, error: lastError };
+}
+
 async function inspectA2ajCoverage(fetchImpl, target, timeoutMs) {
   const groups = await Promise.all(
     [
@@ -247,13 +299,9 @@ async function inspectOntarioRuntime(fetchImpl, target, timeoutMs) {
   try {
     const parsed = new URL(sourceUrl ?? "");
     const match = parsed.pathname.match(
-      /\/laws\/(statute|regulation)\/([a-z0-9.-]+)/i,
+      /\/laws\/(?:api\/v2\/legislation\/en\/doc-search\/)?(statute|regulation)\/([a-z0-9.-]+)$/i,
     );
-    if (
-      parsed.protocol !== "https:" ||
-      parsed.hostname !== "www.ontario.ca" ||
-      !match
-    )
+    if (parsed.protocol !== "https:" || parsed.hostname !== "www.ontario.ca" || !match)
       throw new Error("invalid Ontario source URL");
     officialPath = `${match[1].toLowerCase()}/${match[2].toLowerCase()}`;
   } catch {
@@ -361,41 +409,62 @@ export async function observeLiveLegalSources({
   now = () => new Date(),
   clock = () => Date.now(),
   timeoutMs = 10_000,
+  retryAttempts = DEFAULT_RETRY_ATTEMPTS,
+  retryDelayMs = DEFAULT_RETRY_DELAY_MS,
+  sleepImpl = wait,
 } = {}) {
   const observedAt = now().toISOString();
   const providers = {};
+  const boundedRetryAttempts = Number.isInteger(retryAttempts)
+    ? Math.max(1, retryAttempts)
+    : DEFAULT_RETRY_ATTEMPTS;
+  const boundedRetryDelayMs = Number.isFinite(retryDelayMs)
+    ? Math.max(0, retryDelayMs)
+    : DEFAULT_RETRY_DELAY_MS;
 
-  for (const target of REQUIRED_TARGETS) {
+  for (const target of LIVE_TARGETS) {
     const startedAt = clock();
+    let attempts = 1;
     try {
-      const observation =
-        target.kind === "a2aj-split-coverage"
-          ? await inspectA2ajCoverage(fetchImpl, target, timeoutMs)
-          : target.kind === "ontario-runtime"
-            ? await inspectOntarioRuntime(fetchImpl, target, timeoutMs)
-            : target.kind === "justice-runtime"
-              ? await inspectJusticeRuntime(fetchImpl, target, timeoutMs)
-              : await (async () => {
-                  const response = await fetchResponse(
-                    fetchImpl,
-                    target.url,
-                    target,
-                    timeoutMs,
-                  );
-                  const fallbackVersion = await inspect(response, target);
-                  return {
-                    sourceVersion: responseVersion(response, fallbackVersion),
-                  };
-                })();
+      const outcome = await observeWithRetries(
+        target,
+        () =>
+          target.kind === "a2aj-split-coverage"
+            ? inspectA2ajCoverage(fetchImpl, target, timeoutMs)
+            : target.kind === "ontario-runtime"
+              ? inspectOntarioRuntime(fetchImpl, target, timeoutMs)
+              : target.kind === "justice-runtime"
+                ? inspectJusticeRuntime(fetchImpl, target, timeoutMs)
+                : (async () => {
+                    const response = await fetchResponse(
+                      fetchImpl,
+                      target.url,
+                      target,
+                      timeoutMs,
+                    );
+                    const fallbackVersion = await inspect(response, target);
+                    return {
+                      sourceVersion: responseVersion(response, fallbackVersion),
+                    };
+                  })(),
+        {
+          retryAttempts: boundedRetryAttempts,
+          retryDelayMs: boundedRetryDelayMs,
+          sleepImpl,
+        },
+      );
+      attempts = outcome.attempts;
+      if (!outcome.ok) throw outcome.error;
       providers[target.id] = {
         state: "healthy",
         checkedAt: observedAt,
         lastSuccessfulAt: observedAt,
         consecutiveFailures: 0,
         consecutiveSuccesses: 1,
-        sourceVersion: observation.sourceVersion,
+        sourceVersion: outcome.observation.sourceVersion,
         latencyClass: latencyClass(Math.max(0, clock() - startedAt)),
         reasonCode: "ok",
+        attempts: outcome.attempts,
       };
     } catch (error) {
       providers[target.id] = {
@@ -407,6 +476,7 @@ export async function observeLiveLegalSources({
         sourceVersion: null,
         latencyClass: latencyClass(Math.max(0, clock() - startedAt)),
         reasonCode: reasonCode(error),
+        attempts,
       };
     }
   }
@@ -432,16 +502,24 @@ export async function observeLiveLegalSources({
     reasonCode: "licensed-connector-disabled",
   };
 
-  const requiredHealthy = REQUIRED_TARGETS.every(
+  const requiredHealthy = LIVE_TARGETS.filter(({ required }) => required).every(
     ({ id }) => providers[id]?.state === "healthy",
   );
   return {
-    version: "1.0.0",
+    version: "1.1.0",
     observedAt,
     liveChecksPerformed: true,
     status: requiredHealthy ? "healthy" : "degraded",
+    requiredProviderIds: requiredLiveProviderIds,
+    optionalProviderIds: optionalLiveProviderIds,
     providers,
   };
 }
 
-export const requiredLiveProviderIds = REQUIRED_TARGETS.map(({ id }) => id);
+export const requiredLiveProviderIds = LIVE_TARGETS.filter(
+  ({ required }) => required,
+).map(({ id }) => id);
+export const optionalLiveProviderIds = LIVE_TARGETS.filter(
+  ({ required }) => !required,
+).map(({ id }) => id);
+export const observedLiveProviderIds = LIVE_TARGETS.map(({ id }) => id);