diff --git a/mcp_backend/src/migrations/192_backfill_indexed_articles.sql b/mcp_backend/src/migrations/192_backfill_indexed_articles.sql new file mode 100644 index 00000000..b9d4d77b --- /dev/null +++ b/mcp_backend/src/migrations/192_backfill_indexed_articles.sql @@ -0,0 +1,90 @@ +-- LEXAI-1957: in-force articles whose number carries an index were unreachable. +-- +-- get_legislation_section with an explicit number goes to PG, and the adapter +-- filters `la.is_current = true`. Measured on prod 2026-08-18, counting only +-- articles present in the CURRENT edition per npa.article (max ed_date), for +-- acts in public.legislation: +-- +-- indexed («350-1»): 1 707 in force, 541 reachable — 1 166 missing (68.3%) +-- plain («350») : 15 982 in force, 15 472 reachable — 510 missing (3.2%) +-- +-- Of the 1 166, only 148 had a row that merely lacked the flag; 1 018 had no row +-- at all. So this is not a flag backfill — the rows were never written. +-- +-- Root cause: Rada writes the index with spaces around the hyphen — the stored +-- ЦПК heading is «Стаття 350 - 1 .». import-historical-editions.ts matched +-- `(\d+(?:-\d+)?)`, which allows none, so it captured «350», collided with the +-- real article 350 and lost the row to ON CONFLICT DO NOTHING. That is why every +-- ЦПК edition from 2004 to 2028 held exactly 500 articles and not one indexed, +-- against 525 in npa.article. The importer now builds its regexes from +-- ARTICLE_NUMBER_PATTERN, so new imports keep the index. +-- +-- This migration repairs what is already stored. npa.article is the source of +-- truth: 2 208 156 rows, verified against the raw text one for one. Missing +-- articles are attached to each act's OWN current snapshot date, so an act keeps +-- exactly one current edition rather than gaining a second. +-- +-- Idempotent: the NOT EXISTS guard and ON CONFLICT DO NOTHING make a re-run a +-- no-op. Skips cleanly where the npa schema is absent (local without the corpus). + +DO $$ +DECLARE + inserted_count INTEGER; +BEGIN + IF to_regclass('npa.article') IS NULL THEN + RAISE NOTICE 'LEXAI-1957: npa.article absent, skipping'; + RETURN; + END IF; + + WITH snap AS ( -- each act's own current snapshot date + SELECT l.id AS legislation_id, lower(l.rada_id) AS nreg, max(la.version_date) AS vd + FROM legislation l + JOIN legislation_articles la ON la.legislation_id = l.id AND la.is_current + GROUP BY 1, 2 + ), ed AS ( -- the in-force edition in the clean corpus + SELECT nreg, max(ed_date) AS ed FROM npa.article GROUP BY 1 + ), plan AS ( + SELECT s.legislation_id, s.vd, a.art_no, a.title, a.body + FROM snap s + JOIN ed e ON e.nreg = s.nreg + JOIN npa.article a ON a.nreg = s.nreg AND a.ed_date = e.ed + WHERE NOT EXISTS ( + SELECT 1 FROM legislation_articles x + WHERE x.legislation_id = s.legislation_id + AND x.article_number = a.art_no + AND x.is_current + ) + ) + INSERT INTO legislation_articles + (legislation_id, article_number, title, full_text, byte_size, is_current, version_date, metadata) + SELECT legislation_id, art_no, nullif(btrim(coalesce(title, '')), ''), body, + octet_length(body), true, vd, + jsonb_build_object('source', 'npa.article', 'task', 'LEXAI-1957') + FROM plan + ON CONFLICT (legislation_id, article_number, version_date) DO NOTHING; + + GET DIAGNOSTICS inserted_count = ROW_COUNT; + RAISE NOTICE 'LEXAI-1957: inserted % in-force articles', inserted_count; + + -- Match the table's own convention: existing rows do not repeat the heading in + -- full_text (only 3.9% do), and title is a short line, not a paragraph. Taking + -- npa's body verbatim left «Стаття 350 - 1 .» in the text and a 284-character + -- "title", which surfaced in the MCP payload as a truncated sentence. + UPDATE legislation_articles la + SET full_text = s.stripped, + byte_size = octet_length(s.stripped), + title = nullif(btrim(s.first_line), ''), + updated_at = now() + FROM ( + SELECT id, + regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', '') AS stripped, + CASE + WHEN length(split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1)) <= 200 + THEN split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1) + END AS first_line + FROM legislation_articles + WHERE metadata->>'task' = 'LEXAI-1957' + AND full_text ~ '^\s*Стаття\s' + ) s + WHERE la.id = s.id; +END $$; diff --git a/mcp_backend/src/services/__tests__/act-number.test.ts b/mcp_backend/src/services/__tests__/act-number.test.ts index 4b05affa..981268a0 100644 --- a/mcp_backend/src/services/__tests__/act-number.test.ts +++ b/mcp_backend/src/services/__tests__/act-number.test.ts @@ -18,6 +18,7 @@ import { normalizeArticleNumber, looksLikeOfficialNumber, pickActNumber, + ARTICLE_NUMBER_PATTERN, type ActNumberMatch, } from '../act-number.js'; @@ -154,3 +155,32 @@ describeDb('npa.norm_number parity (requires DATABASE_URL)', () => { expect(drift).toEqual([]); }); }); + +describe('ARTICLE_NUMBER_PATTERN', () => { + // Rada writes the index hyphen with spaces around it and sometimes as an en + // dash — the stored ЦПК heading is «Стаття 350 - 1 .». The historical-editions + // importer allowed neither, captured «350», collided with the real article 350 + // and lost the row; 1 018 in-force indexed articles ended up with no row at all + // (LEXAI-1957). This guards the shipped pattern, not a copy of it. + const rx = new RegExp(`Стаття\\s+(${ARTICLE_NUMBER_PATTERN})`); + + it.each([ + ['Стаття 350. Рішення', '350'], + ['Стаття 350-1. Підсудність', '350-1'], + ['Стаття 350 - 1 . Підсудність', '350 - 1'], + ['Стаття 350–1. Підсудність', '350–1'], + ['Стаття 350 — 1. Підсудність', '350 — 1'], + ])('captures the whole number in %s', (heading, expected) => { + expect(rx.exec(heading)?.[1]).toBe(expected); + }); + + it('every spelling normalises to the stored form', () => { + const stored = ['Стаття 350-1.', 'Стаття 350 - 1 .', 'Стаття 350–1.', 'Стаття 350 — 1.'] + .map((h) => normalizeArticleNumber(rx.exec(h)![1])); + expect(new Set(stored)).toEqual(new Set(['350-1'])); + }); + + it('does not swallow the next article number', () => { + expect(rx.exec('Стаття 350. Текст')?.[1]).toBe('350'); + }); +}); diff --git a/mcp_backend/src/services/act-number.ts b/mcp_backend/src/services/act-number.ts index 5b4c8d34..1f2a1a16 100644 --- a/mcp_backend/src/services/act-number.ts +++ b/mcp_backend/src/services/act-number.ts @@ -202,6 +202,20 @@ export function looksLikeOfficialNumber(raw: string): boolean { * empty, and it never backtracks to the longer branch — «стаття 111» came out * as «аття111» and matched no article at all. */ +/** + * Article-number sub-pattern, as it appears in a heading — «350», «350-1», + * «350 - 1», «350–1». Rada surrounds the index hyphen with spaces and sometimes + * writes it as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .». + * + * Anything that reads an article number out of a heading builds its regex from + * this, so there is one definition to keep right. The historical-editions + * importer used a pattern that allowed neither spaces nor the dash variants, so + * it captured «350», collided with the real article 350 and lost the row to + * ON CONFLICT DO NOTHING — 1 018 in-force indexed articles had no row at all + * (LEXAI-1957). Pair it with normalizeArticleNumber to get the stored form. + */ +export const ARTICLE_NUMBER_PATTERN = String.raw`\d+(?:\s*[-–—]\s*\d+)?`; + export function normalizeArticleNumber(raw: string): string { return String(raw ?? '') .replace(/^\s*(стаття|статті|статтею|ст|пункт|пп|п)\.?\s*/i, '') diff --git a/scripts/rada/import-historical-editions.ts b/scripts/rada/import-historical-editions.ts index f2533306..93484898 100644 --- a/scripts/rada/import-historical-editions.ts +++ b/scripts/rada/import-historical-editions.ts @@ -19,6 +19,7 @@ import * as path from 'path'; import * as http from 'http'; import * as https from 'https'; import pg from 'pg'; +import { ARTICLE_NUMBER_PATTERN, normalizeArticleNumber } from '../../mcp_backend/src/services/act-number.js'; // ─── Config ────────────────────────────────────────────────────────────────── @@ -217,10 +218,19 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s const seen = new Set(); // Try 
 format first (historical editions)
-  const preBoldRegex = /Стаття\s+(\d+(?:-\d+)?)\.?<\/b>\s*(.*?)(?=Стаття\s+\d|<\/pre>\s*$|$)/gs;
+// The index is separated by a hyphen that Rada often surrounds with spaces and
+// sometimes writes as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .».
+// The old pattern allowed neither, so it captured «350», collided with the real
+// article 350 and was dropped by ON CONFLICT DO NOTHING — which is why every ЦПК
+// edition from 2004 to 2028 held exactly 500 articles and not one with an index,
+// against 525 in npa.article. 1 018 in-force indexed articles had no row at all
+// (LEXAI-1957). normalizeArticleNumber folds the dashes and strips the spaces so
+// the stored value matches npa.article.art_no character for character.
+  const preBoldRegex = new RegExp(
+    `Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?\\s*([\\s\\S]*?)(?=Стаття\\s+\\d|
\\s*$|$)`, 'g'); let match; while ((match = preBoldRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (seen.has(artNum)) continue; seen.add(artNum); @@ -247,9 +257,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s // Fallback: format (current /print pages) seen.clear(); articles.length = 0; - const rvtsRegex = /\s*Стаття\s+(\d+(?:-\d+)?)\.?\s*([^<]*)<\/span>\s*(.*?)(?=\s*Стаття\s+\d|$)/gs; + const rvtsRegex = new RegExp( + `\\s*Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?\\s*([^<]*)\\s*([\\s\\S]*?)(?=\\s*Стаття\\s+\\d|$)`, 'g'); while ((match = rvtsRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (seen.has(artNum)) continue; seen.add(artNum); @@ -272,9 +283,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s if (articles.length < 3) { seen.clear(); articles.length = 0; - const plainRegex = /Стаття\s+(\d+(?:-\d+)?)\.\s*([^\n]{3,200})/g; + const plainRegex = new RegExp( + `Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.\\s*([^\\n]{3,200})`, 'g'); while ((match = plainRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (!seen.has(artNum)) { seen.add(artNum); articles.push({