fix(legislation): keep the article index when importing editions

mcp_backend/src/migrations/192_backfill_indexed_articles.sql

@@ -0,0 +1,90 @@
+-- LEXAI-1957: in-force articles whose number carries an index were unreachable.
+--
+-- get_legislation_section with an explicit number goes to PG, and the adapter
+-- filters `la.is_current = true`. Measured on prod 2026-08-18, counting only
+-- articles present in the CURRENT edition per npa.article (max ed_date), for
+-- acts in public.legislation:
+--
+--   indexed («350-1»): 1 707 in force, 541 reachable — 1 166 missing (68.3%)
+--   plain   («350»)  : 15 982 in force, 15 472 reachable — 510 missing (3.2%)
+--
+-- Of the 1 166, only 148 had a row that merely lacked the flag; 1 018 had no row
+-- at all. So this is not a flag backfill — the rows were never written.
+--
+-- Root cause: Rada writes the index with spaces around the hyphen — the stored
+-- ЦПК heading is «Стаття 350 - 1 .». import-historical-editions.ts matched
+-- `(\d+(?:-\d+)?)`, which allows none, so it captured «350», collided with the
+-- real article 350 and lost the row to ON CONFLICT DO NOTHING. That is why every
+-- ЦПК edition from 2004 to 2028 held exactly 500 articles and not one indexed,
+-- against 525 in npa.article. The importer now builds its regexes from
+-- ARTICLE_NUMBER_PATTERN, so new imports keep the index.
+--
+-- This migration repairs what is already stored. npa.article is the source of
+-- truth: 2 208 156 rows, verified against the raw text one for one. Missing
+-- articles are attached to each act's OWN current snapshot date, so an act keeps
+-- exactly one current edition rather than gaining a second.
+--
+-- Idempotent: the NOT EXISTS guard and ON CONFLICT DO NOTHING make a re-run a
+-- no-op. Skips cleanly where the npa schema is absent (local without the corpus).
+
+DO $$
+DECLARE
+  inserted_count INTEGER;
+BEGIN
+  IF to_regclass('npa.article') IS NULL THEN
+    RAISE NOTICE 'LEXAI-1957: npa.article absent, skipping';
+    RETURN;
+  END IF;
+
+  WITH snap AS (        -- each act's own current snapshot date
+    SELECT l.id AS legislation_id, lower(l.rada_id) AS nreg, max(la.version_date) AS vd
+    FROM legislation l
+    JOIN legislation_articles la ON la.legislation_id = l.id AND la.is_current
+    GROUP BY 1, 2
+  ), ed AS (            -- the in-force edition in the clean corpus
+    SELECT nreg, max(ed_date) AS ed FROM npa.article GROUP BY 1
+  ), plan AS (
+    SELECT s.legislation_id, s.vd, a.art_no, a.title, a.body
+    FROM snap s
+    JOIN ed e ON e.nreg = s.nreg
+    JOIN npa.article a ON a.nreg = s.nreg AND a.ed_date = e.ed
+    WHERE NOT EXISTS (
+      SELECT 1 FROM legislation_articles x
+      WHERE x.legislation_id = s.legislation_id
+        AND x.article_number = a.art_no
+        AND x.is_current
+    )
+  )
+  INSERT INTO legislation_articles
+    (legislation_id, article_number, title, full_text, byte_size, is_current, version_date, metadata)
+  SELECT legislation_id, art_no, nullif(btrim(coalesce(title, '')), ''), body,
+         octet_length(body), true, vd,
+         jsonb_build_object('source', 'npa.article', 'task', 'LEXAI-1957')
+  FROM plan
+  ON CONFLICT (legislation_id, article_number, version_date) DO NOTHING;
+
+  GET DIAGNOSTICS inserted_count = ROW_COUNT;
+  RAISE NOTICE 'LEXAI-1957: inserted % in-force articles', inserted_count;
+
+  -- Match the table's own convention: existing rows do not repeat the heading in
+  -- full_text (only 3.9% do), and title is a short line, not a paragraph. Taking
+  -- npa's body verbatim left «Стаття 350 - 1 .» in the text and a 284-character
+  -- "title", which surfaced in the MCP payload as a truncated sentence.
+  UPDATE legislation_articles la
+  SET full_text  = s.stripped,
+      byte_size  = octet_length(s.stripped),
+      title      = nullif(btrim(s.first_line), ''),
+      updated_at = now()
+  FROM (
+    SELECT id,
+           regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', '') AS stripped,
+           CASE
+             WHEN length(split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1)) <= 200
+             THEN split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1)
+           END AS first_line
+    FROM legislation_articles
+    WHERE metadata->>'task' = 'LEXAI-1957'
+      AND full_text ~ '^\s*Стаття\s'
+  ) s
+  WHERE la.id = s.id;
+END $$;

mcp_backend/src/services/__tests__/act-number.test.ts

@@ -18,6 +18,7 @@ import {
   normalizeArticleNumber,
   looksLikeOfficialNumber,
   pickActNumber,
+  ARTICLE_NUMBER_PATTERN,
   type ActNumberMatch,
 } from '../act-number.js';
 
@@ -154,3 +155,32 @@ describeDb('npa.norm_number parity (requires DATABASE_URL)', () => {
     expect(drift).toEqual([]);
   });
 });
+
+describe('ARTICLE_NUMBER_PATTERN', () => {
+  // Rada writes the index hyphen with spaces around it and sometimes as an en
+  // dash — the stored ЦПК heading is «Стаття 350 - 1 .». The historical-editions
+  // importer allowed neither, captured «350», collided with the real article 350
+  // and lost the row; 1 018 in-force indexed articles ended up with no row at all
+  // (LEXAI-1957). This guards the shipped pattern, not a copy of it.
+  const rx = new RegExp(`Стаття\\s+(${ARTICLE_NUMBER_PATTERN})`);
+
+  it.each([
+    ['Стаття 350. Рішення', '350'],
+    ['Стаття 350-1. Підсудність', '350-1'],
+    ['Стаття 350 - 1 . Підсудність', '350 - 1'],
+    ['Стаття 350–1. Підсудність', '350–1'],
+    ['Стаття 350 — 1. Підсудність', '350 — 1'],
+  ])('captures the whole number in %s', (heading, expected) => {
+    expect(rx.exec(heading)?.[1]).toBe(expected);
+  });
+
+  it('every spelling normalises to the stored form', () => {
+    const stored = ['Стаття 350-1.', 'Стаття 350 - 1 .', 'Стаття 350–1.', 'Стаття 350 — 1.']
+      .map((h) => normalizeArticleNumber(rx.exec(h)![1]));
+    expect(new Set(stored)).toEqual(new Set(['350-1']));
+  });
+
+  it('does not swallow the next article number', () => {
+    expect(rx.exec('Стаття 350. Текст')?.[1]).toBe('350');
+  });
+});

mcp_backend/src/services/act-number.ts

@@ -202,6 +202,20 @@ export function looksLikeOfficialNumber(raw: string): boolean {
  * empty, and it never backtracks to the longer branch — «стаття 111» came out
  * as «аття111» and matched no article at all.
  */
+/**
+ * Article-number sub-pattern, as it appears in a heading — «350», «350-1»,
+ * «350 - 1», «350–1». Rada surrounds the index hyphen with spaces and sometimes
+ * writes it as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .».
+ *
+ * Anything that reads an article number out of a heading builds its regex from
+ * this, so there is one definition to keep right. The historical-editions
+ * importer used a pattern that allowed neither spaces nor the dash variants, so
+ * it captured «350», collided with the real article 350 and lost the row to
+ * ON CONFLICT DO NOTHING — 1 018 in-force indexed articles had no row at all
+ * (LEXAI-1957). Pair it with normalizeArticleNumber to get the stored form.
+ */
+export const ARTICLE_NUMBER_PATTERN = String.raw`\d+(?:\s*[-–—]\s*\d+)?`;
+
 export function normalizeArticleNumber(raw: string): string {
   return String(raw ?? '')
     .replace(/^\s*(стаття|статті|статтею|ст|пункт|пп|п)\.?\s*/i, '')

scripts/rada/import-historical-editions.ts

@@ -19,6 +19,7 @@ import * as path from 'path';
 import * as http from 'http';
 import * as https from 'https';
 import pg from 'pg';
+import { ARTICLE_NUMBER_PATTERN, normalizeArticleNumber } from '../../mcp_backend/src/services/act-number.js';
 
 // ─── Config ──────────────────────────────────────────────────────────────────
 
@@ -217,10 +218,19 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s
   const seen = new Set<string>();
 
   // Try <pre><b> format first (historical editions)
-  const preBoldRegex = /<b>Стаття\s+(\d+(?:-\d+)?)\.?<\/b>\s*(.*?)(?=<b>Стаття\s+\d|<\/pre>\s*$|$)/gs;
+// The index is separated by a hyphen that Rada often surrounds with spaces and
+// sometimes writes as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .».
+// The old pattern allowed neither, so it captured «350», collided with the real
+// article 350 and was dropped by ON CONFLICT DO NOTHING — which is why every ЦПК
+// edition from 2004 to 2028 held exactly 500 articles and not one with an index,
+// against 525 in npa.article. 1 018 in-force indexed articles had no row at all
+// (LEXAI-1957). normalizeArticleNumber folds the dashes and strips the spaces so
+// the stored value matches npa.article.art_no character for character.
+  const preBoldRegex = new RegExp(
+    `<b>Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?</b>\\s*([\\s\\S]*?)(?=<b>Стаття\\s+\\d|</pre>\\s*$|$)`, 'g');
   let match;
   while ((match = preBoldRegex.exec(html)) !== null) {
-    const artNum = match[1].trim();
+    const artNum = normalizeArticleNumber(match[1]);
     if (seen.has(artNum)) continue;
     seen.add(artNum);
 
@@ -247,9 +257,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s
   // Fallback: <span class=rvts9> format (current /print pages)
   seen.clear();
   articles.length = 0;
-  const rvtsRegex = /<span\s+class=["']?rvts9["']?>\s*Стаття\s+(\d+(?:-\d+)?)\.?\s*([^<]*)<\/span>\s*(.*?)(?=<span\s+class=["']?rvts9["']?>\s*Стаття\s+\d|$)/gs;
+  const rvtsRegex = new RegExp(
+    `<span\\s+class=["']?rvts9["']?>\\s*Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?\\s*([^<]*)</span>\\s*([\\s\\S]*?)(?=<span\\s+class=["']?rvts9["']?>\\s*Стаття\\s+\\d|$)`, 'g');
   while ((match = rvtsRegex.exec(html)) !== null) {
-    const artNum = match[1].trim();
+    const artNum = normalizeArticleNumber(match[1]);
     if (seen.has(artNum)) continue;
     seen.add(artNum);
 
@@ -272,9 +283,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s
   if (articles.length < 3) {
     seen.clear();
     articles.length = 0;
-    const plainRegex = /Стаття\s+(\d+(?:-\d+)?)\.\s*([^\n]{3,200})/g;
+    const plainRegex = new RegExp(
+      `Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.\\s*([^\\n]{3,200})`, 'g');
     while ((match = plainRegex.exec(html)) !== null) {
-      const artNum = match[1].trim();
+      const artNum = normalizeArticleNumber(match[1]);
       if (!seen.has(artNum)) {
         seen.add(artNum);
         articles.push({