From 29c8cb42df1bfb52a601a5e1916fd130c9c16c20 Mon Sep 17 00:00:00 2001 From: vovkes Date: Tue, 18 Aug 2026 15:50:56 +0200 Subject: [PATCH] fix(legislation): keep the article index when importing editions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rada writes an indexed article number with spaces around the hyphen: the stored ЦПК heading is «Стаття 350 - 1 .». import-historical-editions.ts matched `(\d+(?:-\d+)?)`, which allows neither the spaces nor the en/em dash variants, so it captured «350», collided with the real article 350 and lost the row to ON CONFLICT DO NOTHING. Every ЦПК edition from 2004 to 2028 therefore held exactly 500 articles and not one with an index, against 525 in npa.article. The user-visible effect: get_legislation_section with an explicit number goes to PG and filters is_current, so «ст. 350-1 ЦПК» answered "not found" although the article is in force. Counting only articles present in the current edition per npa.article, for acts in public.legislation: indexed 1 707 in force but 541 reachable (68.3% missing), against 3.2% for plain ones. Of the 1 166, just 148 had a row lacking the flag — 1 018 had no row at all. ARTICLE_NUMBER_PATTERN now sits beside normalizeArticleNumber and all three importer regexes are built from it, so there is one definition rather than four spellings that drift. Migration 192 repairs what is already stored, attaching missing in-force articles to each act's OWN current snapshot date so an act keeps exactly one current edition; it is idempotent and skips where the npa schema is absent. Applied to prod: 1 344 rows (1 149 indexed), after which indexed reachability went 68.3% missing → 1.0%, plain 3.2% → 2.0%, and the live MCP call for 1618-15 / 350-1 returns «Підсудність». Re-running the migration inserts 0. The test builds its regex from the shipped constant instead of restating it, and was checked against the old pattern: it fails with «350 - 1» → «350». Co-Authored-By: Claude Opus 5 (1M context) --- .../192_backfill_indexed_articles.sql | 90 +++++++++++++++++++ .../src/services/__tests__/act-number.test.ts | 30 +++++++ mcp_backend/src/services/act-number.ts | 14 +++ scripts/rada/import-historical-editions.ts | 24 +++-- 4 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 mcp_backend/src/migrations/192_backfill_indexed_articles.sql diff --git a/mcp_backend/src/migrations/192_backfill_indexed_articles.sql b/mcp_backend/src/migrations/192_backfill_indexed_articles.sql new file mode 100644 index 000000000..b9d4d77be --- /dev/null +++ b/mcp_backend/src/migrations/192_backfill_indexed_articles.sql @@ -0,0 +1,90 @@ +-- LEXAI-1957: in-force articles whose number carries an index were unreachable. +-- +-- get_legislation_section with an explicit number goes to PG, and the adapter +-- filters `la.is_current = true`. Measured on prod 2026-08-18, counting only +-- articles present in the CURRENT edition per npa.article (max ed_date), for +-- acts in public.legislation: +-- +-- indexed («350-1»): 1 707 in force, 541 reachable — 1 166 missing (68.3%) +-- plain («350») : 15 982 in force, 15 472 reachable — 510 missing (3.2%) +-- +-- Of the 1 166, only 148 had a row that merely lacked the flag; 1 018 had no row +-- at all. So this is not a flag backfill — the rows were never written. +-- +-- Root cause: Rada writes the index with spaces around the hyphen — the stored +-- ЦПК heading is «Стаття 350 - 1 .». import-historical-editions.ts matched +-- `(\d+(?:-\d+)?)`, which allows none, so it captured «350», collided with the +-- real article 350 and lost the row to ON CONFLICT DO NOTHING. That is why every +-- ЦПК edition from 2004 to 2028 held exactly 500 articles and not one indexed, +-- against 525 in npa.article. The importer now builds its regexes from +-- ARTICLE_NUMBER_PATTERN, so new imports keep the index. +-- +-- This migration repairs what is already stored. npa.article is the source of +-- truth: 2 208 156 rows, verified against the raw text one for one. Missing +-- articles are attached to each act's OWN current snapshot date, so an act keeps +-- exactly one current edition rather than gaining a second. +-- +-- Idempotent: the NOT EXISTS guard and ON CONFLICT DO NOTHING make a re-run a +-- no-op. Skips cleanly where the npa schema is absent (local without the corpus). + +DO $$ +DECLARE + inserted_count INTEGER; +BEGIN + IF to_regclass('npa.article') IS NULL THEN + RAISE NOTICE 'LEXAI-1957: npa.article absent, skipping'; + RETURN; + END IF; + + WITH snap AS ( -- each act's own current snapshot date + SELECT l.id AS legislation_id, lower(l.rada_id) AS nreg, max(la.version_date) AS vd + FROM legislation l + JOIN legislation_articles la ON la.legislation_id = l.id AND la.is_current + GROUP BY 1, 2 + ), ed AS ( -- the in-force edition in the clean corpus + SELECT nreg, max(ed_date) AS ed FROM npa.article GROUP BY 1 + ), plan AS ( + SELECT s.legislation_id, s.vd, a.art_no, a.title, a.body + FROM snap s + JOIN ed e ON e.nreg = s.nreg + JOIN npa.article a ON a.nreg = s.nreg AND a.ed_date = e.ed + WHERE NOT EXISTS ( + SELECT 1 FROM legislation_articles x + WHERE x.legislation_id = s.legislation_id + AND x.article_number = a.art_no + AND x.is_current + ) + ) + INSERT INTO legislation_articles + (legislation_id, article_number, title, full_text, byte_size, is_current, version_date, metadata) + SELECT legislation_id, art_no, nullif(btrim(coalesce(title, '')), ''), body, + octet_length(body), true, vd, + jsonb_build_object('source', 'npa.article', 'task', 'LEXAI-1957') + FROM plan + ON CONFLICT (legislation_id, article_number, version_date) DO NOTHING; + + GET DIAGNOSTICS inserted_count = ROW_COUNT; + RAISE NOTICE 'LEXAI-1957: inserted % in-force articles', inserted_count; + + -- Match the table's own convention: existing rows do not repeat the heading in + -- full_text (only 3.9% do), and title is a short line, not a paragraph. Taking + -- npa's body verbatim left «Стаття 350 - 1 .» in the text and a 284-character + -- "title", which surfaced in the MCP payload as a truncated sentence. + UPDATE legislation_articles la + SET full_text = s.stripped, + byte_size = octet_length(s.stripped), + title = nullif(btrim(s.first_line), ''), + updated_at = now() + FROM ( + SELECT id, + regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', '') AS stripped, + CASE + WHEN length(split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1)) <= 200 + THEN split_part(regexp_replace(full_text, '^\s*Стаття\s+[0-9]+(\s*[-–—]\s*[0-9]+)?\s*\.?\s*', ''), chr(10), 1) + END AS first_line + FROM legislation_articles + WHERE metadata->>'task' = 'LEXAI-1957' + AND full_text ~ '^\s*Стаття\s' + ) s + WHERE la.id = s.id; +END $$; diff --git a/mcp_backend/src/services/__tests__/act-number.test.ts b/mcp_backend/src/services/__tests__/act-number.test.ts index 4b05affae..981268a02 100644 --- a/mcp_backend/src/services/__tests__/act-number.test.ts +++ b/mcp_backend/src/services/__tests__/act-number.test.ts @@ -18,6 +18,7 @@ import { normalizeArticleNumber, looksLikeOfficialNumber, pickActNumber, + ARTICLE_NUMBER_PATTERN, type ActNumberMatch, } from '../act-number.js'; @@ -154,3 +155,32 @@ describeDb('npa.norm_number parity (requires DATABASE_URL)', () => { expect(drift).toEqual([]); }); }); + +describe('ARTICLE_NUMBER_PATTERN', () => { + // Rada writes the index hyphen with spaces around it and sometimes as an en + // dash — the stored ЦПК heading is «Стаття 350 - 1 .». The historical-editions + // importer allowed neither, captured «350», collided with the real article 350 + // and lost the row; 1 018 in-force indexed articles ended up with no row at all + // (LEXAI-1957). This guards the shipped pattern, not a copy of it. + const rx = new RegExp(`Стаття\\s+(${ARTICLE_NUMBER_PATTERN})`); + + it.each([ + ['Стаття 350. Рішення', '350'], + ['Стаття 350-1. Підсудність', '350-1'], + ['Стаття 350 - 1 . Підсудність', '350 - 1'], + ['Стаття 350–1. Підсудність', '350–1'], + ['Стаття 350 — 1. Підсудність', '350 — 1'], + ])('captures the whole number in %s', (heading, expected) => { + expect(rx.exec(heading)?.[1]).toBe(expected); + }); + + it('every spelling normalises to the stored form', () => { + const stored = ['Стаття 350-1.', 'Стаття 350 - 1 .', 'Стаття 350–1.', 'Стаття 350 — 1.'] + .map((h) => normalizeArticleNumber(rx.exec(h)![1])); + expect(new Set(stored)).toEqual(new Set(['350-1'])); + }); + + it('does not swallow the next article number', () => { + expect(rx.exec('Стаття 350. Текст')?.[1]).toBe('350'); + }); +}); diff --git a/mcp_backend/src/services/act-number.ts b/mcp_backend/src/services/act-number.ts index 5b4c8d34d..1f2a1a161 100644 --- a/mcp_backend/src/services/act-number.ts +++ b/mcp_backend/src/services/act-number.ts @@ -202,6 +202,20 @@ export function looksLikeOfficialNumber(raw: string): boolean { * empty, and it never backtracks to the longer branch — «стаття 111» came out * as «аття111» and matched no article at all. */ +/** + * Article-number sub-pattern, as it appears in a heading — «350», «350-1», + * «350 - 1», «350–1». Rada surrounds the index hyphen with spaces and sometimes + * writes it as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .». + * + * Anything that reads an article number out of a heading builds its regex from + * this, so there is one definition to keep right. The historical-editions + * importer used a pattern that allowed neither spaces nor the dash variants, so + * it captured «350», collided with the real article 350 and lost the row to + * ON CONFLICT DO NOTHING — 1 018 in-force indexed articles had no row at all + * (LEXAI-1957). Pair it with normalizeArticleNumber to get the stored form. + */ +export const ARTICLE_NUMBER_PATTERN = String.raw`\d+(?:\s*[-–—]\s*\d+)?`; + export function normalizeArticleNumber(raw: string): string { return String(raw ?? '') .replace(/^\s*(стаття|статті|статтею|ст|пункт|пп|п)\.?\s*/i, '') diff --git a/scripts/rada/import-historical-editions.ts b/scripts/rada/import-historical-editions.ts index f25333064..934848980 100644 --- a/scripts/rada/import-historical-editions.ts +++ b/scripts/rada/import-historical-editions.ts @@ -19,6 +19,7 @@ import * as path from 'path'; import * as http from 'http'; import * as https from 'https'; import pg from 'pg'; +import { ARTICLE_NUMBER_PATTERN, normalizeArticleNumber } from '../../mcp_backend/src/services/act-number.js'; // ─── Config ────────────────────────────────────────────────────────────────── @@ -217,10 +218,19 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s const seen = new Set(); // Try 
 format first (historical editions)
-  const preBoldRegex = /Стаття\s+(\d+(?:-\d+)?)\.?<\/b>\s*(.*?)(?=Стаття\s+\d|<\/pre>\s*$|$)/gs;
+// The index is separated by a hyphen that Rada often surrounds with spaces and
+// sometimes writes as an en/em dash: the stored ЦПК heading is «Стаття 350 - 1 .».
+// The old pattern allowed neither, so it captured «350», collided with the real
+// article 350 and was dropped by ON CONFLICT DO NOTHING — which is why every ЦПК
+// edition from 2004 to 2028 held exactly 500 articles and not one with an index,
+// against 525 in npa.article. 1 018 in-force indexed articles had no row at all
+// (LEXAI-1957). normalizeArticleNumber folds the dashes and strips the spaces so
+// the stored value matches npa.article.art_no character for character.
+  const preBoldRegex = new RegExp(
+    `Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?\\s*([\\s\\S]*?)(?=Стаття\\s+\\d|
\\s*$|$)`, 'g'); let match; while ((match = preBoldRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (seen.has(artNum)) continue; seen.add(artNum); @@ -247,9 +257,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s // Fallback: format (current /print pages) seen.clear(); articles.length = 0; - const rvtsRegex = /\s*Стаття\s+(\d+(?:-\d+)?)\.?\s*([^<]*)<\/span>\s*(.*?)(?=\s*Стаття\s+\d|$)/gs; + const rvtsRegex = new RegExp( + `\\s*Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.?\\s*([^<]*)\\s*([\\s\\S]*?)(?=\\s*Стаття\\s+\\d|$)`, 'g'); while ((match = rvtsRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (seen.has(artNum)) continue; seen.add(artNum); @@ -272,9 +283,10 @@ function extractArticlesFromEditionHtml(html: string): Array<{ article_number: s if (articles.length < 3) { seen.clear(); articles.length = 0; - const plainRegex = /Стаття\s+(\d+(?:-\d+)?)\.\s*([^\n]{3,200})/g; + const plainRegex = new RegExp( + `Стаття\\s+(${ARTICLE_NUMBER_PATTERN})\\.\\s*([^\\n]{3,200})`, 'g'); while ((match = plainRegex.exec(html)) !== null) { - const artNum = match[1].trim(); + const artNum = normalizeArticleNumber(match[1]); if (!seen.has(artNum)) { seen.add(artNum); articles.push({