diff --git a/mcp_backend/src/migrations/190_pl_legislation.sql b/mcp_backend/src/migrations/190_pl_legislation.sql new file mode 100644 index 00000000..b6ce0b0f --- /dev/null +++ b/mcp_backend/src/migrations/190_pl_legislation.sql @@ -0,0 +1,205 @@ +-- Migration 190: Polish legislation register and amendment graph (Sejm ELI API) +-- +-- Source: https://api.sejm.gov.pl/eli, no auth, JSON. 164,213 acts as of +-- 2026-08-14: Dziennik Ustaw (DU) 97,681 and Monitor Polski (MP) 66,532, +-- enumerated from 197 year listings plus one detail call each. +-- +-- Shape follows nl_laws / nl_law_editions (migration 181), which in turn follows +-- the Ukrainian legislation_editions, rather than inventing a third vocabulary +-- for the same idea. The verdict-code idiom on the text side comes from +-- npa.edition. Tables live in public alongside pl_court_decisions (migration +-- 151); the separate `npa` schema exists only because that corpus had to shadow +-- live `legislation*` tables until cutover, and there is no Polish incumbent. +-- +-- What Poland does differently, and what this schema is shaped around: there is +-- no point-in-time service. The only texts that exist are the one published on +-- promulgation ("tekst ogloszony", served under the act's own ELI) and one per +-- "obwieszczenie w sprawie ogloszenia jednolitego tekstu" (each served under the +-- OBWIESZCZENIE's ELI, not the base act's). Verified: DU/1974/141/text.html is +-- the 1974 Kodeks pracy, 8 hits for "socjalistyczn"; DU/2020/1320/text.html is +-- the 2020 consolidation, 0 hits for "socjalistyczn" and 12 for "monitoring". +-- +-- So this schema stores published snapshots and the full amendment edge set, and +-- refuses to interpolate between them. See pl_act_snapshots. + +-- --------------------------------------------------------------------------- +-- One row per ELI address. Includes amending acts and obwieszczenia: they are +-- acts, they have their own Dz.U. position, and excluding them would break the +-- reference graph. is_consolidation / consolidates_eli exist so that "how many +-- Polish laws are there" is not answered by counting a law once per +-- consolidation. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS pl_acts ( + eli TEXT PRIMARY KEY, -- 'DU/1964/93' + publisher TEXT NOT NULL, -- DU | MP + year INTEGER NOT NULL, + pos INTEGER NOT NULL, + volume INTEGER, + address TEXT, -- 'WDU19640160093' + display_address TEXT, -- 'Dz.U. 1964 nr 16 poz. 93' + act_type TEXT, -- Ustawa | Rozporzadzenie | Obwieszczenie ... + title TEXT, + previous_titles TEXT[], + status TEXT, -- 'akt posiada tekst jednolity' ... + in_force TEXT, -- IN_FORCE | NOT_IN_FORCE | NULL + announcement_date DATE, -- data aktu + promulgation DATE, -- data ogloszenia w dzienniku + entry_into_force DATE, + valid_from DATE, + repeal_date DATE, + expiration_date DATE, + -- "stan prawny na dzien". Set on consolidating obwieszczenia, NULL on base + -- acts. This is the field that makes the temporal answer computable instead + -- of guessed; see pl_act_snapshots.exact_on. + legal_status_date DATE, + change_date TIMESTAMPTZ, -- source-side last modification; drives resync + text_html BOOLEAN NOT NULL DEFAULT false, + text_pdf BOOLEAN NOT NULL DEFAULT false, + texts JSONB, -- [{fileName,type}], type H|O|I|U|T + keywords TEXT[], + keywords_names TEXT[], + released_by TEXT[], + obligated TEXT[], + authorized_body TEXT[], + directives JSONB, -- EU directives implemented + prints JSONB, -- Sejm print numbers + -- Derived by build_pl_snapshots.sql from pl_act_references, NOT from the + -- title. 12% of DU obwieszczenia are not consolidations at all, and title + -- matching would additionally have to survive 1930s orthography. + is_consolidation BOOLEAN NOT NULL DEFAULT false, -- has 'Tekst jednolity dla aktu' + consolidates_eli TEXT, -- the base act it consolidates + amends_count INTEGER NOT NULL DEFAULT 0, + amended_by_count INTEGER NOT NULL DEFAULT 0, + snapshot_count INTEGER NOT NULL DEFAULT 0, -- meaningful on base acts only + detail_fetched_at TIMESTAMPTZ, + imported_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_pl_acts_pub_year ON pl_acts (publisher, year, pos); +CREATE INDEX IF NOT EXISTS idx_pl_acts_type ON pl_acts (act_type); +CREATE INDEX IF NOT EXISTS idx_pl_acts_status ON pl_acts (status); +CREATE INDEX IF NOT EXISTS idx_pl_acts_keywords ON pl_acts USING GIN (keywords); +CREATE INDEX IF NOT EXISTS idx_pl_acts_consol ON pl_acts (consolidates_eli) + WHERE consolidates_eli IS NOT NULL; +-- the resync worklist: acts whose source-side changeDate moved past our fetch +CREATE INDEX IF NOT EXISTS idx_pl_acts_changed ON pl_acts (change_date DESC); +-- the text-harvest worklist is an anti-join filtered on this +CREATE INDEX IF NOT EXISTS idx_pl_acts_html ON pl_acts (eli) WHERE text_html; +-- the Stage-1 worklist +CREATE INDEX IF NOT EXISTS idx_pl_acts_nodetail ON pl_acts (eli) + WHERE detail_fetched_at IS NULL; + +-- --------------------------------------------------------------------------- +-- The reference graph, one row per edge, category kept verbatim in Polish. +-- +-- Categories observed so far: +-- Akty zmieniajace / Akty zmienione (amendment, both directions) +-- Akty uchylajace / Akty uchylone / Akty uznane za uchylone +-- Inf. o tekscie jednolitym / Tekst jednolity dla aktu +-- Nowelizacje po tekscie jednolitym +-- Akty wykonawcze / Podstawa prawna / Podstawa prawna z art. +-- Przepisy wprowadzajace / Uchylenia wynikajace z / Odeslania +-- Orzeczenie TK / Orzeczenie TK dla aktu / Sprostowanie +-- +-- Storing the label verbatim rather than a normalised enum is deliberate: the +-- source adds categories, and an edge whose category we do not yet understand +-- must survive the load instead of being silently dropped. +-- +-- These edges are read from the act detail payload, which inlines them. The +-- separate /references endpoint returns the same set - verified byte-identical +-- on DU/1964/93 across all five categories (223/223, 113/113, 11/11, 25/25, +-- 1/1) - so it is not fetched, saving 164,213 requests. +-- +-- effective_date is the API's `date` on an entry. On 'Akty zmieniajace' it is +-- the amendment's entry into force and is what pl_article_as_of() builds its +-- answer from. Populated on 100% of the 455 edges checked across KC/KP/KK/KPA, +-- and it carries future dates (KC has one at 2028-11-01). +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS pl_act_references ( + src_eli TEXT NOT NULL, -- act on whose detail payload the entry appeared + category TEXT NOT NULL, + dst_eli TEXT NOT NULL, + effective_date DATE, + art_ref TEXT, -- 'Podstawa prawna z art.' carries a provision string + imported_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (src_eli, category, dst_eli) +); + +CREATE INDEX IF NOT EXISTS idx_pl_refs_dst ON pl_act_references (dst_eli, category); +-- the hot path: "amendments to act X effective in (a, b]" +CREATE INDEX IF NOT EXISTS idx_pl_refs_amend + ON pl_act_references (src_eli, effective_date) + WHERE category = 'Akty zmieniające'; + +-- --------------------------------------------------------------------------- +-- THE TEMPORAL TABLE. One row per published text of one law. +-- +-- act_eli the law's identity: ALWAYS the base act, never the obwieszczenie +-- snapshot_eli the act that physically published this text +-- +-- For the original text the two are equal. For a consolidated text snapshot_eli +-- is the obwieszczenie and act_eli is the target of its 'Tekst jednolity dla +-- aktu' reference. So counting laws is count(DISTINCT act_eli) and counting +-- texts is count(*), and neither can be got wrong by accident. +-- +-- exact_on is the ONE date on which this text is exactly the law in force: +-- ogloszony -> entry_into_force, falling back to promulgation +-- jednolity -> legal_status_date, falling back to announcement_date +-- The API publishes legal_status_date precisely so this is not a guess. Ten of +-- the eleven Kodeks pracy snapshots carry it; DU/1998/94 does not, hence the +-- fallback and the exact_on_src column recording which was used. +-- +-- valid_to is DERIVED as the next snapshot's exact_on minus one day, NOT taken +-- from the source's expirationDate. They disagree systematically and on +-- purpose: expirationDate is the date the OBWIESZCZENIE was superseded, which +-- is the next one's promulgation, so consecutive consolidated texts overlap by +-- weeks or months (KP: DU/2019/1040 expires 2020-07-30 but DU/2020/1320 is +-- exact from 2020-06-18). Overlapping intervals cannot answer "which text for +-- date D"; max(exact_on) <= D can. source_expiration is kept beside it so the +-- disagreement is auditable rather than quietly resolved. +-- +-- drift_from is the honesty column: the effective date of the first amendment +-- landing after exact_on. On [exact_on, drift_from) the text is exact. On +-- [drift_from, valid_to] it is the nearest published text and is known to be +-- behind by amendments_after edges. There is no third state and no +-- reconstruction. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS pl_act_snapshots ( + act_eli TEXT NOT NULL, + snapshot_eli TEXT NOT NULL, + seq INTEGER NOT NULL, -- 0.. in exact_on order + snapshot_kind TEXT NOT NULL, -- ogloszony | jednolity + exact_on DATE, + exact_on_src TEXT, -- entryIntoForce | promulgation | + -- legalStatusDate | announcementDate | unknown + valid_to DATE, -- NULL = latest published snapshot + source_expiration DATE, + drift_from DATE, + amendments_after INTEGER NOT NULL DEFAULT 0, + has_html BOOLEAN NOT NULL DEFAULT false, + text_url TEXT, -- .../acts/{snapshot_eli}/text.html + pdf_file TEXT, -- fallback: type T or U filename + PRIMARY KEY (act_eli, snapshot_eli) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_pl_snap_seq ON pl_act_snapshots (act_eli, seq); +CREATE INDEX IF NOT EXISTS idx_pl_snap_asof ON pl_act_snapshots (act_eli, exact_on DESC); +CREATE INDEX IF NOT EXISTS idx_pl_snap_eli ON pl_act_snapshots (snapshot_eli); +CREATE INDEX IF NOT EXISTS idx_pl_snap_nohtml ON pl_act_snapshots (act_eli) WHERE NOT has_html; + +-- --------------------------------------------------------------------------- +-- pl_court_decisions predates this work (migration 151) and was loaded from +-- three snapshot sources with no natural key: row ids were built from the +-- position of the row inside a parquet file ("hf-pl-court-raw-12098-train_08_of_9"), +-- so a new dataset revision duplicates the corpus instead of updating it, and +-- the loader used ON CONFLICT DO NOTHING, so bad text could never be repaired. +-- +-- judgment_id is Portal Orzeczen's own identifier, e.g. +-- "152515050001006_II_K_000202_2017_Uz_2017-06-27_001". It is what makes the +-- import idempotent and what lets SAOS and pl-court-raw rows describing the +-- same judgment be collapsed. The UNIQUE index is created CONCURRENTLY in the +-- companion file, because this table is 105 GB. +-- --------------------------------------------------------------------------- +ALTER TABLE pl_court_decisions ADD COLUMN IF NOT EXISTS judgment_id TEXT; +ALTER TABLE pl_court_decisions ADD COLUMN IF NOT EXISTS text_status TEXT; diff --git a/mcp_backend/src/migrations/191_pl_law_texts.sql b/mcp_backend/src/migrations/191_pl_law_texts.sql new file mode 100644 index 00000000..0f884045 --- /dev/null +++ b/mcp_backend/src/migrations/191_pl_law_texts.sql @@ -0,0 +1,238 @@ +-- Migration 191: Polish legislation text, article by article. +-- +-- 39,110 of 97,681 Dziennik Ustaw acts serve text.html (40.0%). Monitor Polski +-- serves none - 0 of 66,532, checked across all 92 MP year listings - so MP is +-- a register-and-graph corpus and no text pipeline is built for it. DU coverage +-- by era: 1918-1989 11.3%, 1990-1999 15.7%, 2000-2011 16.9%, 2012-2023 99.7%, +-- 2024 100%, 2025-2026 0%. The recent zero is a publication lag, not a +-- permanent hole, which is why sync_eli_changes.py must re-poll acts previously +-- recorded as HTML-less instead of trusting text_html once. +-- +-- The API also exposes /struct, a nested tree whose node ids are byte-identical +-- to the id= attribute on the
wrappers in the HTML. +-- Extraction therefore walks struct and looks each node up in the DOM, rather +-- than pattern-matching headings the way the Ukrainian splitter had to. That is +-- not a stylistic preference: DU/2020/1320 carries three
inside the obwieszczenie's own footnotes, which struct +-- correctly omits, so a regex over the same HTML returns 497 articles for a +-- 494-article code. +-- +-- The article PK ends in `ord` rather than the article number, for the reason +-- nl_law_articles gives: labels repeat. DU/1964/93 lists +-- book_trzecia-titl_XI-bran_I-arti_538 twice (2,290 struct nodes, 2,289 +-- distinct ids) and DU/1997/553 lists `none_` twice. Keying on the label would +-- reject the whole insert batch. + +-- --------------------------------------------------------------------------- +-- One row per article per snapshot. `text` includes the article's own +-- paragraf / ustep / punkt / litera children, because that is the unit Polish +-- practice cites and quotes. Sub-article addressing is served by pl_act_units, +-- which stores character offsets into this text rather than a second copy of it. +-- +-- symbol vs struct_id: struct ids are NOT stable across snapshots. Art. 415 KC +-- is book_trzecia-titl_VI-arti_415 in the 1964 text and +-- book_TRZECIA-titl_VI-arti_415 in the 2023 consolidation - the book name +-- changes case. Cross-snapshot article identity therefore keys on art_no, +-- derived from the snapshot-local symbol, never on the path. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS pl_act_articles ( + act_eli TEXT NOT NULL, + snapshot_eli TEXT NOT NULL, + ord INTEGER NOT NULL, -- document order within the snapshot + symbol TEXT NOT NULL, -- 'arti_304_4' - stable across snapshots + struct_id TEXT NOT NULL, -- 'bran_PIETNASTY-arti_304_4' - snapshot-local + art_no TEXT NOT NULL, -- '304^4' - canonical form + art_display TEXT NOT NULL, -- 'Art. 304(4).' - as rendered + art_sort_1 INTEGER, -- 304 + art_sort_2 INTEGER, -- 4, NULL when there is no superscript + art_title TEXT, -- article heading where one exists + text TEXT NOT NULL, + n_chars INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (act_eli, snapshot_eli, ord) +); + +-- "what did art. 415 of the civil code say" - the lookup that matters +CREATE INDEX IF NOT EXISTS idx_pl_articles_lookup + ON pl_act_articles (act_eli, art_no, snapshot_eli); +CREATE INDEX IF NOT EXISTS idx_pl_articles_snapshot + ON pl_act_articles (snapshot_eli, ord); +CREATE INDEX IF NOT EXISTS idx_pl_articles_symbol + ON pl_act_articles (act_eli, symbol); + +-- --------------------------------------------------------------------------- +-- The struct tree, all levels, no text. Two jobs: +-- 1. navigation: ksiega / czesc / tytul / dzial / rozdzial / oddzial headings +-- 2. sub-article addressing: "art. 415 § 1" resolves to a (char_from, char_to) +-- slice of pl_act_articles.text, so no provision is stored twice. +-- Node types seen: part, book, titl, bran, chpt, schp, arti, para, pint, lett, +-- pass, none. +-- +-- in_annex marks nodes under a top-level part whose title starts with +-- "Zalacznik". On a consolidating obwieszczenie the law IS the annex: DU/2020/1320 +-- puts all 494 arti nodes under part_2 ("Zalacznik - Tekst jednolity ustawy ... +-- Kodeks pracy") and none under part_1 ("Tresc obwieszczenia"). +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS pl_act_units ( + snapshot_eli TEXT NOT NULL, + ord INTEGER NOT NULL, -- preorder position in the struct tree + parent_ord INTEGER, + depth SMALLINT NOT NULL, + struct_id TEXT NOT NULL, + symbol TEXT, + unit_type TEXT NOT NULL, + name TEXT, + title TEXT, + article_ord INTEGER, -- pl_act_articles.ord of the enclosing arti + char_from INTEGER, -- offset into that article's text + char_to INTEGER, + in_annex BOOLEAN NOT NULL DEFAULT false, + PRIMARY KEY (snapshot_eli, ord) +); + +CREATE INDEX IF NOT EXISTS idx_pl_units_art + ON pl_act_units (snapshot_eli, article_ord) WHERE article_ord IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_pl_units_type + ON pl_act_units (snapshot_eli, unit_type); + +-- --------------------------------------------------------------------------- +-- The fetch record. EVERY snapshot gets a row, success or not, so that "never +-- downloaded" and "downloaded, came back empty" are never the same state - the +-- rule migration 182 set for nl_law_edition_texts. +-- +-- http_status carries our verdicts above the HTTP range, the npa.edition idiom: +-- 200 ok +-- 404 gone +-- 599 network failure / timeout after all retries +-- 900 HTTP 200 with a zero-byte body <- the dominant Polish failure mode +-- 901 HTTP 200, non-empty, but no

200; +CREATE INDEX IF NOT EXISTS idx_pl_snapshot_texts_act + ON pl_snapshot_texts (act_eli); +CREATE INDEX IF NOT EXISTS idx_pl_snapshot_texts_hash + ON pl_snapshot_texts (text_hash) WHERE text_hash IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- The only sanctioned way to answer "what did article X say on date D". +-- +-- It returns the nearest published snapshot at or before the date AND the +-- amendments that took effect between that snapshot and the date. There is no +-- argument that suppresses the second half, because a caller who sees only the +-- text would reasonably believe it was the law on that date, and for Poland +-- that is usually false. +-- +-- exact the snapshot is dated at or before D and no amendment took +-- effect in between - the text IS the law as at D +-- stale the nearest snapshot is followed by N amendments effective +-- by D; the text is the closest published one and is behind +-- no_text a snapshot exists but its text was never published in a +-- machine-readable form (verdict 903) +-- pre_enactment D precedes the act's first published text +-- +-- Verified live on 2026-08-14: +-- pl_article_as_of('DU/1974/141','1','2019-06-01') +-- -> exact, DU/2019/1040, exact_on 2019-05-09, 0 amendments in between +-- pl_article_as_of('DU/1974/141','1','2020-06-01') +-- -> stale, with a non-empty amendment list +-- --------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION pl_article_as_of( + p_act TEXT, p_art_no TEXT, p_date DATE) +RETURNS TABLE ( + confidence TEXT, + snapshot_eli TEXT, + snapshot_kind TEXT, + exact_on DATE, + days_before_query INTEGER, + art_no TEXT, + art_display TEXT, + text TEXT, + amendments_since JSONB, + next_snapshot_eli TEXT, + next_snapshot_on DATE +) LANGUAGE sql STABLE AS $fn$ +WITH s AS ( + SELECT * FROM pl_act_snapshots + WHERE act_eli = p_act AND exact_on IS NOT NULL AND exact_on <= p_date + ORDER BY exact_on DESC, seq DESC + LIMIT 1 +), nx AS ( + SELECT * FROM pl_act_snapshots + WHERE act_eli = p_act AND exact_on > p_date + ORDER BY exact_on ASC, seq ASC + LIMIT 1 +), am AS ( + SELECT jsonb_agg(jsonb_build_object( + 'eli', r.dst_eli, + 'effective_date', r.effective_date, + 'title', a.title) ORDER BY r.effective_date) AS j, + count(*) AS n + FROM pl_act_references r + LEFT JOIN pl_acts a ON a.eli = r.dst_eli + WHERE r.src_eli = p_act + AND r.category = 'Akty zmieniające' + AND r.effective_date > (SELECT exact_on FROM s) + AND r.effective_date <= p_date +) +SELECT + CASE WHEN art.text IS NULL THEN 'no_text' + WHEN coalesce((SELECT n FROM am), 0) = 0 THEN 'exact' + ELSE 'stale' END, + s.snapshot_eli, s.snapshot_kind, s.exact_on, + (p_date - s.exact_on)::int, + art.art_no, art.art_display, art.text, + coalesce((SELECT j FROM am), '[]'::jsonb), + (SELECT snapshot_eli FROM nx), (SELECT exact_on FROM nx) +FROM s +LEFT JOIN pl_act_articles art + ON art.snapshot_eli = s.snapshot_eli + AND art.art_no = p_art_no +UNION ALL +-- No snapshot at or before the date: say so, rather than returning zero rows, +-- which a caller can too easily read as "no such article". +SELECT 'pre_enactment', NULL, NULL, NULL, NULL, p_art_no, NULL, NULL, + '[]'::jsonb, + (SELECT snapshot_eli FROM nx), (SELECT exact_on FROM nx) +WHERE NOT EXISTS (SELECT 1 FROM s); +$fn$; + +-- Per-act chain, for display and for the audit's chain-integrity check. +CREATE OR REPLACE VIEW pl_act_timeline AS +SELECT s.act_eli, a.title, s.seq, s.snapshot_eli, s.snapshot_kind, + s.exact_on, s.exact_on_src, s.drift_from, s.valid_to, + s.source_expiration, s.amendments_after, s.has_html, + t.http_status, t.article_count +FROM pl_act_snapshots s +LEFT JOIN pl_acts a ON a.eli = s.act_eli +LEFT JOIN pl_snapshot_texts t ON t.snapshot_eli = s.snapshot_eli +ORDER BY s.act_eli, s.seq; diff --git a/scripts/pl/192_pl_indexes_concurrently.sql b/scripts/pl/192_pl_indexes_concurrently.sql new file mode 100644 index 00000000..29a13064 --- /dev/null +++ b/scripts/pl/192_pl_indexes_concurrently.sql @@ -0,0 +1,76 @@ +-- Companion to migrations 190/191. NOT run by mcp_backend/src/migrations/migrate.ts. +-- +-- migrate.ts executes each file as one db.query(), i.e. inside one implicit +-- transaction, and CREATE INDEX CONCURRENTLY cannot run in a transaction. The +-- established convention is to park those builds here and apply them by hand; +-- the precedent is scripts/nl/179b_nl_decisions_indexes_concurrently.sql. +-- +-- Apply on prod: +-- docker cp scripts/pl/192_pl_indexes_concurrently.sql secondlayer-postgres-prod:/tmp/ +-- docker exec secondlayer-postgres-prod psql -U secondlayer -d secondlayer_prod \ +-- -f /tmp/192_pl_indexes_concurrently.sql +-- +-- Then ALWAYS check for a build that was interrupted, which leaves an INVALID +-- index that silently does not serve queries: +-- SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid; + +-- --------------------------------------------------------------------------- +-- pl_court_decisions is 2,864,093 rows / 105 GB. Every index on it must be +-- CONCURRENTLY or the table is locked for the duration. +-- --------------------------------------------------------------------------- + +-- The dedup and idempotency key. Built as a plain index first: the table today +-- has 2.86M rows with judgment_id NULL, and a UNIQUE index over them is fine +-- (NULLs do not collide in Postgres), but the backfill in 40_repair_legacy.py +-- must run before anything relies on uniqueness being meaningful. +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_court_judgment_id + ON pl_court_decisions (judgment_id) WHERE judgment_id IS NOT NULL; + +-- Incremental passes and the substrate /changes feed. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_court_updated_at + ON pl_court_decisions (updated_at); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_court_text_status + ON pl_court_decisions (text_status) WHERE text_status IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- Legislation. +-- --------------------------------------------------------------------------- + +-- Polish stemming: Postgres ships no 'polish' text search configuration by +-- default, and the existing idx_pl_court_fts (migration 151) silently uses +-- 'simple', i.e. no morphology at all - the same defect the Dutch corpus had +-- until 179b replaced 'simple' with 'dutch'. +-- +-- Check what this cluster actually has BEFORE running the FTS index below: +-- SELECT cfgname FROM pg_ts_config ORDER BY 1; +-- If a 'polish' config exists, change 'simple' to 'polish' in the two statements +-- below and in the pl_court_decisions rebuild at the bottom. If it does not, +-- installing it needs a hunspell pl_PL dictionary in the image, which is a +-- deployment change and therefore a separate decision - record which branch was +-- taken here rather than leaving the choice implicit. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_articles_fts + ON pl_act_articles USING GIN (to_tsvector('simple', text)); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_acts_title_trgm + ON pl_acts USING GIN (title gin_trgm_ops); + +-- Article ordering for structural navigation: 304^4 sorts after 304 and before 305. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pl_articles_sort + ON pl_act_articles (act_eli, art_sort_1, art_sort_2); + +-- --------------------------------------------------------------------------- +-- Deferred: the pl_court_decisions FTS rebuild. +-- +-- idx_pl_court_fts is a GIN index on to_tsvector('simple', ...) over 105 GB of +-- text. Rebuilding it with a real Polish configuration is a multi-hour build and +-- a large amount of new disk, so it is deliberately NOT in this file. Do it as +-- its own scheduled operation once the corpus is repaired and deduplicated - +-- rebuilding it now would index rows that 40_repair_legacy.py is about to +-- collapse. +-- +-- CREATE INDEX CONCURRENTLY idx_pl_court_fts_pl ON pl_court_decisions +-- USING GIN (to_tsvector('polish', coalesce(parties,'') || ' ' || +-- coalesce(abstract,'') || ' ' || coalesce(full_text,''))); +-- DROP INDEX CONCURRENTLY idx_pl_court_fts; +-- --------------------------------------------------------------------------- diff --git a/scripts/pl/README.md b/scripts/pl/README.md new file mode 100644 index 00000000..e445cd14 --- /dev/null +++ b/scripts/pl/README.md @@ -0,0 +1,133 @@ +# Polish corpus + +Court decisions and legislation-with-history for Poland, into `secondlayer_prod`. + +Everything except the database write runs on **local.lex** (16 cores, 123 GB RAM, +PyMuPDF/tesseract/pdftotext present). Prod is not used for fetching or parsing: +it keeps the harvest off the prod EIP pool, and traffic goes local → AWS, which +is ingress rather than paid egress. + +``` +PL_SSH_HOST=prod # "" when running on the prod host itself +PG_CONTAINER=secondlayer-postgres-prod +PG_USER=secondlayer +PG_DB=secondlayer_prod +``` + +## State + +| stage | script | status | +|---|---|---| +| 0 pin baseline | `pin_baseline.py` | **done** - runs in 89 s, 0 problems | +| 1 register + graph | `harvest_eli_register.py` | **done** - 164,213 acts, 732,983 edges, 0 failures | +| 2 snapshot chain | `build_pl_snapshots.sql` | **done** - 164,206 snapshots, 154,843 laws, 4,223 with >1 | +| 3+4 struct + text | `harvest_eli_texts.py` | **done** - 673,735 articles, 5.5M units, 7 unresolved of 39,106 | +| 5 incremental sync | `sync_eli_changes.py` | not written | +| 6 audit | `audit_pl_load.sh` | not written | +| courts | `harvest_ncourt.py`, `harvest_cbosa.py`, `harvest_sn_tk.py` | not written | +| legacy repair | `repair_legacy.py` | not written | + +Parser: `pl_article_parser.py`, tests `test_pl_article_parser.py` (all passing). +Schema: `mcp_backend/src/migrations/190_pl_legislation.sql`, `191_pl_law_texts.sql` +(applied to prod 2026-08-14; 184/185 were already taken, hence 190/191), +plus `192_pl_indexes_concurrently.sql` **which the migration runner must not run** +(it wraps each file in one transaction and `CREATE INDEX CONCURRENTLY` cannot run +in one - same reason as `scripts/nl/179b_*`). + +## Run + +```bash +python3 scripts/pl/pin_baseline.py --out /data/pl_eli/baseline +python3 scripts/pl/harvest_eli_register.py --listings +python3 scripts/pl/harvest_eli_register.py --details --limit 500 # smoke first +python3 scripts/pl/harvest_eli_register.py --details +psql -f scripts/pl/build_pl_snapshots.sql +FIXTURES=/data/pl_eli/fixtures python3 scripts/pl/test_pl_article_parser.py --fetch +``` + +`WORKERS=4 RATE_MS=250` is ~6 req/s. The source sustained 11.5 req/s on details +and 4.1 req/s on `text.html` across ~700 probe requests with no 429, so this is +margin, not a limit it imposed. Details pass ≈ 7-8 h, text pass ≈ 3-4 h. + +## What the source actually does + +Measured 2026-08-14, not taken from documentation. + +**There is no point-in-time service.** Only two kinds of text exist: the one +published on promulgation, and one per *obwieszczenie w sprawie ogłoszenia +jednolitego tekstu*. The consolidated text is served under the **obwieszczenie's** +ELI, not the base act's - `DU/1974/141/text.html` is the 1974 Kodeks pracy +(8 hits for `socjalistyczn`), `DU/2020/1320/text.html` is the 2020 consolidation +(0 hits). Hence snapshots + amendment graph, and no interpolation. + +**`legalStatusDate`** ("stan prawny na dzień") is on consolidating obwieszczenia +and is what makes the temporal answer computable. Absent on the oldest ones +(KP's `DU/1998/94`), hence `exact_on_src`. + +**`valid_to` is derived, not taken.** The source's `expirationDate` is when the +*obwieszczenie* was superseded, so consecutive texts overlap: three of the ten KP +consolidations overlap the next by 28-55 days. `source_expiration` is stored +beside the derived value so the disagreement stays visible. + +**Coverage.** DU 97,681 acts (105 year listings, sum matches the declared +`actsCount`), 39,110 with HTML. By era: 1918-1989 11.3 %, 1990-1999 15.7 %, +2000-2011 16.9 %, 2012-2023 99.7 %, 2024 100 %, **2025-2026 0 %**. That recent +zero is a publication lag, so the incremental sync must re-poll acts recorded as +HTML-less instead of trusting `text_html` once. + +**Monitor Polski has no HTML at all** - 0 of 66,532 across all 92 years. MP is a +register-and-graph corpus; no text pipeline is built for it. + +**`/references` is redundant.** The act detail inlines the same edges, +byte-identical on DU/1964/93 across all five categories (223 / 113 / 11 / 25 / 1). +Not calling it saves 164,213 requests. + +**Konstytucja RP `DU/1997/483` has no machine-readable text** - `textHTML:false`, +`/struct` 404, `text.html` 200-with-zero-bytes, 13.2 MB PDF only. It is carried +as a negative landmark (verdict `903`) rather than an assertion that cannot hold. +Any pipeline assuming "act ⇒ HTML" loses the most recognisable act in the corpus +without noticing. + +## Traps + +- **Charset.** Documents declare it twice in one attribute + (`text/html; charset=UTF-8; charset=UTF-8`) and lxml falls back to latin-1. + Struct ids contain Polish letters (`bran_piąty-chpt_I-arti_114`), so a + mis-decoded id stops matching struct and the article is silently dropped - + 134 of 305 lost on Kodeks pracy while every anchor count still looked right. + Always parse with an explicit `HTMLParser(encoding="utf-8")`. +- **Footnotes.** Rendered inline as + `3)…prose…`. + Left in, they put 21,332 characters of editorial note into the provisions of + DU/2020/1320 (1.6 % of the body), and the marker sitting after an article + number turns `Art. 47` into `Art. 47^6`. +- **Numbering has three levels plus a range form:** `arti_415` → `415`, + `arti_304_4` → `304^4`, `arti_18_3_a` → `18^3a` (24 of 494 articles in the KP + consolidation), `arti_266-280` → `266-280` (a repealed span as one unit, with + an empty body - which is a fact, not a missing value). +- **Struct ids repeat** (DU/1964/93 has 2,290 nodes / 2,289 distinct), so the + article PK ends in `ord` and DOM lookup consumes the k-th occurrence. + Struct ids are also **not stable across snapshots** (`book_trzecia` → + `book_TRZECIA`), so cross-snapshot identity keys on `art_no`, never the path. +- **Do not use `prod_writer.copy_into` for text.** Its `_escape_copy` maps + newline to a space; `plprod.esc` emits a literal `\n` that COPY decodes back. +- **`pl_court_decisions` already holds 2,864,093 rows / 105 GB** from three + snapshot sources. It is stale (hf-pl-nsa stops 2025-02-26), its ids are built + from parquet row positions so a re-import duplicates rather than updates, and + it loaded with `ON CONFLICT DO NOTHING` so bad text cannot be repaired. A + full-table aggregate on it times out at 300 s - shard audits by source/year. + +## Validation rules for article extraction + +- **V1 exact struct coverage.** `extracted == struct declares, in scope`. Not a + ratio. DU/2020/1320 has 497 `unit_arti` anchors for a 494-article code; the + three extras are articles quoted inside the obwieszczenie's own passages, and + no threshold would catch that. +- **V2 label agreement** between the DOM heading and the struct symbol. Catches + mis-pairing of repeated ids. +- **V3 monotonicity, reported not enforced.** It earns its place: DU/1964/93 + labels the article at position 536 `Art. 538.` in **both** struct and the DOM + while its text is the real 536. V2 structurally cannot see that - both sides + agree and are both wrong. A non-zero `nonmonotonic` is a finding to surface, + not a number expected to be zero. +- **V4 substance**: ≥95 % of articles ≥10 chars. diff --git a/scripts/pl/build_pl_snapshots.sql b/scripts/pl/build_pl_snapshots.sql new file mode 100644 index 00000000..b23242b3 --- /dev/null +++ b/scripts/pl/build_pl_snapshots.sql @@ -0,0 +1,152 @@ +-- Stage 2: derive the snapshot chain from the register. Pure SQL, no network. +-- +-- Re-runnable: it rebuilds pl_act_snapshots from pl_acts + pl_act_references +-- every time, so running it again after a details re-fetch simply picks up +-- whatever the register now says. +-- +-- psql -f scripts/pl/build_pl_snapshots.sql +-- +-- What it derives, and why each rule is what it is, is documented in migration +-- 184 next to the columns. The short version: +-- * a law's identity is the BASE act; an obwieszczenie contributes a dated +-- snapshot under that identity, never a law of its own +-- * exact_on is the one date the text is exactly the law +-- * valid_to is derived from the NEXT snapshot's exact_on, not from the +-- source's expirationDate, which overlaps +-- * drift_from / amendments_after say how far behind the text has fallen + +BEGIN; + +-- 1. Consolidation links, taken from the reverse edge the obwieszczenie itself +-- carries. Not from the title: 12% of DU obwieszczenia are not +-- consolidations, and title matching would have to survive 1930s orthography. +UPDATE pl_acts a + SET is_consolidation = false, consolidates_eli = NULL + WHERE a.is_consolidation OR a.consolidates_eli IS NOT NULL; + +UPDATE pl_acts a + SET is_consolidation = true, + consolidates_eli = r.dst_eli + FROM pl_act_references r + WHERE r.src_eli = a.eli + AND r.category = 'Tekst jednolity dla aktu'; + +-- 2. Amendment counters, for display and for the audit. +UPDATE pl_acts a SET amends_count = c.n + FROM (SELECT src_eli, count(*) n FROM pl_act_references + WHERE category = 'Akty zmienione' GROUP BY 1) c + WHERE c.src_eli = a.eli; + +UPDATE pl_acts a SET amended_by_count = c.n + FROM (SELECT src_eli, count(*) n FROM pl_act_references + WHERE category = 'Akty zmieniające' GROUP BY 1) c + WHERE c.src_eli = a.eli; + +-- 3. The snapshot chain. +DELETE FROM pl_act_snapshots; + +WITH raw AS ( + -- The text published on promulgation, under the act's own ELI. Only for + -- acts that are not themselves consolidations of something else. + SELECT a.eli AS act_eli, + a.eli AS snapshot_eli, + 'ogloszony' AS snapshot_kind, + coalesce(a.entry_into_force, a.promulgation, a.announcement_date) AS exact_on, + CASE WHEN a.entry_into_force IS NOT NULL THEN 'entryIntoForce' + WHEN a.promulgation IS NOT NULL THEN 'promulgation' + WHEN a.announcement_date IS NOT NULL THEN 'announcementDate' + ELSE 'unknown' END AS exact_on_src, + a.expiration_date AS source_expiration, + a.text_html AS has_html, + a.texts AS texts + FROM pl_acts a + WHERE NOT a.is_consolidation + + UNION ALL + + -- Each consolidated text, under the BASE act's identity but served from the + -- obwieszczenie's ELI. legalStatusDate is the "stan prawny na dzien" and is + -- the whole reason this can be dated honestly; the oldest obwieszczenia + -- predate the field, hence the fallback. + SELECT c.consolidates_eli, + c.eli, + 'jednolity', + coalesce(c.legal_status_date, c.announcement_date, c.promulgation), + CASE WHEN c.legal_status_date IS NOT NULL THEN 'legalStatusDate' + WHEN c.announcement_date IS NOT NULL THEN 'announcementDate' + WHEN c.promulgation IS NOT NULL THEN 'promulgation' + ELSE 'unknown' END, + c.expiration_date, + c.text_html, + c.texts + FROM pl_acts c + WHERE c.is_consolidation + AND c.consolidates_eli IS NOT NULL + -- A dangling target would otherwise create a snapshot for a law we do + -- not have, and count(DISTINCT act_eli) would then overstate the corpus. + AND EXISTS (SELECT 1 FROM pl_acts b WHERE b.eli = c.consolidates_eli) +), ordered AS ( + SELECT raw.*, + -- Order by exact_on, not by promulgation or by position: KP's + -- DU/2019/1040 was promulgated 2019-05-16 but is exact on + -- 2019-05-09, and only the exact_on order is total and gap-free. + -- NULLs sort last and get no interval, so they can never be picked + -- as "the text in force on D". + row_number() OVER (PARTITION BY act_eli + ORDER BY exact_on NULLS LAST, snapshot_kind DESC, + snapshot_eli) - 1 AS seq, + lead(exact_on) OVER (PARTITION BY act_eli + ORDER BY exact_on NULLS LAST, snapshot_kind DESC, + snapshot_eli) AS next_exact_on + FROM raw +) +INSERT INTO pl_act_snapshots + (act_eli, snapshot_eli, seq, snapshot_kind, exact_on, exact_on_src, + valid_to, source_expiration, has_html, text_url, pdf_file) +SELECT o.act_eli, o.snapshot_eli, o.seq, o.snapshot_kind, o.exact_on, o.exact_on_src, + CASE WHEN o.next_exact_on IS NULL THEN NULL + ELSE o.next_exact_on - 1 END, + o.source_expiration, + o.has_html, + CASE WHEN o.has_html + THEN 'https://api.sejm.gov.pl/eli/acts/' || o.snapshot_eli || '/text.html' + END, + -- The consolidated-text PDF (type U), the fallback when no HTML exists. + (SELECT t->>'fileName' FROM jsonb_array_elements(o.texts) t + WHERE t->>'type' IN ('U', 'T') LIMIT 1) + FROM ordered o + WHERE o.exact_on IS NOT NULL; + +-- 4. The honesty columns. drift_from is the first amendment to land after this +-- text was exact; amendments_after is how many landed before the next +-- published text. A large amendments_after means Poland changed the law +-- repeatedly and published no consolidated text for it - which is exactly +-- what a consumer needs to be told, and exactly what a schema that stored +-- only "current text" would hide. +UPDATE pl_act_snapshots s + SET drift_from = d.first_after, + amendments_after = d.n + FROM ( + SELECT s.act_eli, s.snapshot_eli, + min(r.effective_date) AS first_after, + count(r.*) AS n + FROM pl_act_snapshots s + JOIN pl_act_references r + ON r.src_eli = s.act_eli + AND r.category = 'Akty zmieniające' + AND r.effective_date > s.exact_on + AND r.effective_date <= coalesce(s.valid_to, DATE '9999-12-31') + GROUP BY 1, 2 + ) d + WHERE d.act_eli = s.act_eli AND d.snapshot_eli = s.snapshot_eli; + +-- 5. Roll-ups on the act. +UPDATE pl_acts a SET snapshot_count = c.n + FROM (SELECT act_eli, count(*) n FROM pl_act_snapshots GROUP BY 1) c + WHERE c.act_eli = a.eli; + +COMMIT; + +ANALYZE pl_acts; +ANALYZE pl_act_references; +ANALYZE pl_act_snapshots; diff --git a/scripts/pl/harvest_eli_register.py b/scripts/pl/harvest_eli_register.py new file mode 100644 index 00000000..53949bda --- /dev/null +++ b/scripts/pl/harvest_eli_register.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Stage 1: build the Polish act register and the reference graph. + +Two passes, both resumable, neither with a checkpoint file: + + --listings 197 year listings -> stub rows in pl_acts (the whole register) + --details anti-join "pl_acts WHERE detail_fetched_at IS NULL" -> full rows + plus every edge into pl_act_references + +The worklist IS the database, the rule scripts/nl/harvest_bwb_texts.py states: +a killed run just leaves fewer rows for the next one, and running it twice is a +no-op. Nothing here needs a resume file to be correct. + +The /references endpoint is deliberately NOT called. The act detail payload +inlines the same edges - verified byte-identical on DU/1964/93 across all five +of its categories (223 Akty wykonawcze, 113 Akty zmieniajace, 11 Inf. o tekscie +jednolitym, 25 Orzeczenie TK, 1 Przepisy wprowadzajace) - so calling it would +double the pass for nothing. 164,213 requests saved. + +Runs on local.lex; writes to prod over ssh. Roughly 7-8 h for the details pass +at the default rate. + + python3 scripts/pl/harvest_eli_register.py --listings + python3 scripts/pl/harvest_eli_register.py --details + python3 scripts/pl/harvest_eli_register.py --details --limit 500 # smoke run +""" +import argparse +import json +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import plprod # noqa: E402 + +API = "https://api.sejm.gov.pl/eli" +WORKERS = int(os.environ.get("WORKERS", "4")) +RATE_MS = int(os.environ.get("RATE_MS", "250")) +BATCH = int(os.environ.get("BATCH", "400")) +TRIES = int(os.environ.get("TRIES", "4")) +TIMEOUT = int(os.environ.get("TIMEOUT", "90")) + +ACT_COLS = [ + "eli", "publisher", "year", "pos", "volume", "address", "display_address", + "act_type", "title", "status", "in_force", "announcement_date", + "promulgation", "entry_into_force", "valid_from", "repeal_date", + "expiration_date", "legal_status_date", "change_date", "text_html", + "text_pdf", "texts", "keywords", "keywords_names", "released_by", + "obligated", "authorized_body", "directives", "prints", "detail_fetched_at", +] +REF_COLS = ["src_eli", "category", "dst_eli", "effective_date", "art_ref"] + +_lock = Lock() +_last = [0.0] +counts = {"acts": 0, "refs": 0, "failed": 0} +started = time.time() + + +def throttle(): + gap = RATE_MS / 1000.0 / max(WORKERS, 1) + with _lock: + wait = _last[0] + gap - time.time() + if wait > 0: + time.sleep(wait) + _last[0] = time.time() + + +def fetch_json(path): + """(parsed, http_code). curl rather than urllib: urllib raises + CERTIFICATE_VERIFY_FAILED against this host in some environments.""" + for attempt in range(TRIES): + throttle() + r = subprocess.run( + ["curl", "-s", "-m", str(TIMEOUT), "-w", "\n%{http_code}", f"{API}/{path}"], + capture_output=True) + if r.returncode == 0 and r.stdout: + body, _, code = r.stdout.rpartition(b"\n") + try: + code = int(code) + except ValueError: + code = 0 + if code == 404: + return None, 404 + if code == 200: + if not body: + # A 200 with an empty body is a permanent property of this + # API, not a transient failure. Retrying it would multiply + # the run by the 60% of DU acts that have no HTML. + return None, 900 + try: + return json.loads(body), 200 + except json.JSONDecodeError: + return None, 901 + time.sleep(2 * (attempt + 1)) + return None, 599 + + +def _arr(v): + """Python list -> Postgres text[] literal.""" + if not v: + return None + parts = [] + for x in v: + s = str(x).replace("\\", "\\\\").replace('"', '\\"') + parts.append(f'"{s}"') + return "{" + ",".join(parts) + "}" + + +def _json(v): + return json.dumps(v, ensure_ascii=False) if v else None + + +def act_row(d, fetched=False): + return ( + d.get("ELI"), d.get("publisher"), d.get("year"), d.get("pos"), + d.get("volume"), d.get("address"), d.get("displayAddress"), + d.get("type"), d.get("title"), d.get("status"), d.get("inForce"), + d.get("announcementDate"), d.get("promulgation"), d.get("entryIntoForce"), + d.get("validFrom"), d.get("repealDate"), d.get("expirationDate"), + d.get("legalStatusDate"), d.get("changeDate"), + bool(d.get("textHTML")), bool(d.get("textPDF")), + _json(d.get("texts")), _arr(d.get("keywords")), _arr(d.get("keywordsNames")), + _arr(d.get("releasedBy")), _arr(d.get("obligated")), _arr(d.get("authorizedBody")), + _json(d.get("directives")), _json(d.get("prints")), + time.strftime("%Y-%m-%d %H:%M:%S") if fetched else None, + ) + + +def ref_rows(eli, d): + """Edges from the act's inlined `references` map. + + Category names are kept verbatim in Polish. The source adds categories over + time, and an edge whose category we do not yet recognise must survive the + load rather than be dropped by a normalising enum. + """ + out = [] + for category, entries in (d.get("references") or {}).items(): + seen = set() + for e in entries or []: + # Two shapes exist: the inline form {"id": ..., "date": ...} and the + # /references form {"act": {...}, "date": ...}. Accept both so this + # keeps working if the payload is ever switched. + dst = e.get("id") or (e.get("act") or {}).get("ELI") + if not dst or (category, dst) in seen: + continue + seen.add((category, dst)) + out.append((eli, category, dst, e.get("date"), e.get("art"))) + return out + + +def log(msg): + el = int(time.time() - started) + rate = counts["acts"] / el if el else 0 + print(f"[{el:>6}s] {msg} | acts={counts['acts']} refs={counts['refs']} " + f"failed={counts['failed']} | {rate:.1f} acts/s", flush=True) + + +def do_listings(): + """Enumerate the whole register from the year listings. + + A year listing is a full metadata dump - DU/2020 answers + count == totalCount == 2463 - so the register costs 197 requests, and only + the fields the listing omits (entryIntoForce, legalStatusDate, references, + ...) need a per-act call in the details pass. + """ + total = 0 + for pub in ("DU", "MP"): + meta, code = fetch_json(f"acts/{pub}") + if meta is None: + print(f"FATAL: publisher {pub} -> HTTP {code}", file=sys.stderr) + return 1 + years = list(meta.get("years") or []) + declared = meta.get("actsCount") + seen = 0 + + for year in years: + d, code = fetch_json(f"acts/{pub}/{year}?limit=5000") + if d is None: + print(f" {pub}/{year}: HTTP {code}, skipped", file=sys.stderr) + counts["failed"] += 1 + continue + items = d.get("items") or [] + if d.get("totalCount") != len(items): + # Every later coverage percentage is computed against this + # denominator, so a truncated listing must stop the pass rather + # than quietly shrink the corpus. + print(f"FATAL: {pub}/{year} truncated: totalCount=" + f"{d.get('totalCount')} items={len(items)}", file=sys.stderr) + return 1 + rows = [act_row(i) for i in items if i.get("ELI")] + plprod.upsert_rows("pl_acts", ACT_COLS, rows, ["eli"], prefer_new=True) + counts["acts"] += len(rows) + seen += len(rows) + total += len(rows) + log(f"{pub}/{year}") + + if declared is not None and seen != declared: + print(f"FATAL: {pub} actsCount={declared} but listings gave {seen}", + file=sys.stderr) + return 1 + print(f"{pub}: {seen} acts, matches declared actsCount", flush=True) + + log(f"listings done, {total} acts") + return 0 + + +def handle_detail(eli): + d, code = fetch_json(f"acts/{eli}") + if d is None: + counts["failed"] += 1 + return None, [] + d.setdefault("ELI", eli) + return act_row(d, fetched=True), ref_rows(eli, d) + + +def do_details(limit): + """Fill in what the listing does not carry, and the reference graph.""" + done = 0 + while True: + n = BATCH if not limit else min(BATCH, limit - done) + if n <= 0: + break + work = [r[0] for r in plprod.rows_of( + "SELECT eli FROM pl_acts WHERE detail_fetched_at IS NULL " + f"ORDER BY publisher, year, pos LIMIT {n}")] + if not work: + log("worklist empty, done") + break + + acts, refs = [], [] + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + for row, rrows in pool.map(handle_detail, work): + if row is not None: + acts.append(row) + refs += rrows + + if acts: + plprod.upsert_rows("pl_acts", ACT_COLS, acts, ["eli"], prefer_new=True) + counts["acts"] += len(acts) + if refs: + plprod.upsert_rows("pl_act_references", REF_COLS, refs, + ["src_eli", "category", "dst_eli"], prefer_new=True) + counts["refs"] += len(refs) + + done += len(work) + log(f"batch of {len(work)}") + + if len(acts) == 0: + # Nothing in this batch succeeded, so the same rows come back next + # time and the loop would spin forever. Stop and let the operator + # look, rather than burn the night on a dead endpoint. + print("FATAL: a whole batch failed; stopping instead of looping", + file=sys.stderr) + return 1 + if limit and done >= limit: + break + + log("details done") + return 0 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--listings", action="store_true") + ap.add_argument("--details", action="store_true") + ap.add_argument("--limit", type=int, default=0) + a = ap.parse_args() + if not (a.listings or a.details): + ap.error("pass --listings and/or --details") + rc = 0 + if a.listings: + rc = do_listings() + if rc == 0 and a.details: + rc = do_details(a.limit) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pl/harvest_eli_texts.py b/scripts/pl/harvest_eli_texts.py new file mode 100644 index 00000000..77b32cf9 --- /dev/null +++ b/scripts/pl/harvest_eli_texts.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Stages 3+4: fetch each snapshot's struct and text.html, split into articles. + +One pass over one worklist, not two. The parser needs both documents at the same +time - it walks the struct tree and looks each node up in the DOM - so fetching +them in separate passes would double the resume bookkeeping for no gain. + +Worklist is an anti-join, no checkpoint file: + + pl_act_snapshots LEFT JOIN pl_snapshot_texts ... WHERE the text row is absent + +Every snapshot ends with a pl_snapshot_texts row, success or not, so "never +fetched" and "fetched, came back empty" are never the same state. That also +means a failed snapshot is not retried forever: retrying is an explicit + + DELETE FROM pl_snapshot_texts WHERE http_status IN (599, 900, 902); + +before re-running, which keeps the intent visible instead of burying it in a +loop condition. + +Snapshots the source never published in machine-readable form (has_html = false: +every Monitor Polski act and 58,571 DU acts, including Konstytucja RP) get +verdict 903 written WITHOUT a fetch. 903 is not a failure - it is the honest +statement that no text exists upstream, and it is what separates a gap in the +source from a gap in our harvest. + +Raw documents are staged under --raw before parsing, so changing the extraction +rules is a reparse rather than a refetch. Budget ~3 GB. + + python3 scripts/pl/harvest_eli_texts.py --mark-no-html + python3 scripts/pl/harvest_eli_texts.py --limit 200 # smoke first + python3 scripts/pl/harvest_eli_texts.py + python3 scripts/pl/harvest_eli_texts.py --reparse # no network +""" +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +import lxml.html + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import plprod # noqa: E402 +import pl_article_parser as P # noqa: E402 + +API = "https://api.sejm.gov.pl/eli" +RAW = os.environ.get("RAW", "/data/pl_eli/raw") +WORKERS = int(os.environ.get("WORKERS", "4")) +RATE_MS = int(os.environ.get("RATE_MS", "250")) +BATCH = int(os.environ.get("BATCH", "200")) +TRIES = int(os.environ.get("TRIES", "4")) +TIMEOUT = int(os.environ.get("TIMEOUT", "180")) + +ART_COLS = ["act_eli", "snapshot_eli", "ord", "symbol", "struct_id", "art_no", + "art_display", "art_sort_1", "art_sort_2", "art_title", "text", "n_chars"] +UNIT_COLS = ["snapshot_eli", "ord", "parent_ord", "depth", "struct_id", "symbol", + "unit_type", "name", "title", "article_ord", "char_from", "char_to", + "in_annex"] +TXT_COLS = ["snapshot_eli", "act_eli", "http_status", "html_bytes", "struct_bytes", + "struct_articles", "article_count", "unit_count", "text", "n_chars", + "text_hash", "label_mismatches", "nonmonotonic", "annex_part_id"] + +_lock = Lock() +_last = [0.0] +counts = {"ok": 0, "no_text": 0, "failed": 0, "articles": 0} +started = time.time() + + +def log(msg): + el = int(time.time() - started) + done = counts["ok"] + counts["no_text"] + counts["failed"] + print(f"[{el:>6}s] {msg} | done={done} ok={counts['ok']} " + f"notext={counts['no_text']} failed={counts['failed']} " + f"arts={counts['articles']} | {done/el if el else 0:.1f}/s", flush=True) + + +def throttle(): + gap = RATE_MS / 1000.0 / max(WORKERS, 1) + with _lock: + wait = _last[0] + gap - time.time() + if wait > 0: + time.sleep(wait) + _last[0] = time.time() + + +def raw_path(eli, ext): + pub, year, pos = eli.split("/") + d = os.path.join(RAW, pub, year) + os.makedirs(d, exist_ok=True) + return os.path.join(d, f"{pos}{ext}") + + +def fetch(url, path, use_cache=True): + """(bytes, verdict). Cached on disk so a reparse costs no requests.""" + if use_cache and os.path.exists(path): + with open(path, "rb") as f: + body = f.read() + return (body, P.OK) if body else (b"", 900) + + for attempt in range(TRIES): + throttle() + r = subprocess.run(["curl", "-s", "-m", str(TIMEOUT), "-w", "\n%{http_code}", url], + capture_output=True) + if r.returncode == 0 and r.stdout: + body, _, code = r.stdout.rpartition(b"\n") + try: + code = int(code) + except ValueError: + code = 0 + if code == 404: + return b"", 404 + if code == 200: + # A 200 with an empty body is a permanent property of this API, + # reproduced across acts and symbol forms. Retrying it would + # multiply the run by the acts that have no HTML. + if not body: + return b"", 900 + with open(path, "wb") as f: + f.write(body) + return body, P.OK + time.sleep(2 * (attempt + 1)) + return b"", 599 + + +def _full_text(html): + """Whole-document text, gloss footnotes removed. + + Stored ONLY when no article came out, so a provision is never held twice - + the rule migration 191 inherits from nl_law_edition_texts. + """ + doc = lxml.html.document_fromstring( + html, parser=lxml.html.HTMLParser(encoding="utf-8")) + P.strip_glosses(doc) + return P._clean("".join(doc.itertext())) + + +def handle(row, reparse=False): + act_eli, snap_eli, is_cons = row[0], row[1], row[2] == "t" + + html, hv = fetch(f"{API}/acts/{snap_eli}/text.html", + raw_path(snap_eli, ".html"), use_cache=True) + if hv != P.OK: + counts["failed"] += 1 + return [], [], [(snap_eli, act_eli, hv, len(html), None, None, 0, 0, + None, 0, None, 0, 0, None)] + + sj, sv = fetch(f"{API}/acts/{snap_eli}/struct", + raw_path(snap_eli, ".struct.json"), use_cache=True) + if sv != P.OK: + # The act says it has HTML but publishes no structure. Distinct verdict + # (902) rather than lumping it with a fetch failure: it needs a + # different fix, not a retry. + counts["failed"] += 1 + return [], [], [(snap_eli, act_eli, P.NO_STRUCT, len(html), len(sj), None, + 0, 0, None, 0, None, 0, 0, None)] + + try: + # Not json.loads: ISAP does not escape ASCII double quotes inside string + # values, so any act whose title closes a „ quotation with a straight " + # returns unparseable JSON. DU/2024/561 is 76 KB of valid structure + # behind one such quote, and 140 articles would be lost with it. + struct, _repaired = P.repair_struct_json(sj) + except json.JSONDecodeError: + counts["failed"] += 1 + # Still store the document text. Without a struct there are no articles, + # but losing the act entirely is worse than holding it unsegmented, and + # verdict 901 records exactly which of the two this row is. + full = _full_text(html) + return [], [], [(snap_eli, act_eli, 901, len(html), len(sj), None, 0, 0, + full, len(full), None, 0, 0, None)] + + r = P.parse(struct, html, is_consolidation=is_cons) + + arts = [(act_eli, snap_eli, a["ord"], a["symbol"], a["struct_id"], a["art_no"], + a["art_display"], a["art_sort_1"], a["art_sort_2"], a["art_title"], + a["text"], a["n_chars"]) for a in r.articles] + units = [(snap_eli, u["ord"], u["parent_ord"], u["depth"], u["struct_id"], + u["symbol"], u["unit_type"], u["name"], u["title"], u["article_ord"], + u["char_from"], u["char_to"], u["in_annex"]) for u in r.units] + + joined = "\n".join(a["text"] for a in r.articles) + digest = hashlib.sha256(joined.encode("utf-8")).hexdigest() if joined else None + + # Whole-document text is kept ONLY when no article came out, so a provision + # is never stored twice (the rule migration 182 set for the Dutch corpus). + full = _full_text(html) if not r.articles else None + + if r.verdict == P.OK: + counts["ok"] += 1 + else: + counts["failed"] += 1 + counts["articles"] += len(arts) + + return arts, units, [(snap_eli, act_eli, r.verdict, len(html), len(sj), + r.struct_articles, len(arts), len(units), full, + len(full) if full else 0, digest, r.label_mismatches, + r.nonmonotonic, r.annex_part_id)] + + +def mark_no_html(): + """Write verdict 903 for every snapshot the source never published as HTML. + + Done in SQL, in one statement: it covers ~125k snapshots and there is + nothing to fetch. Re-runnable - the anti-join skips rows already written. + """ + n = plprod.psql(""" + INSERT INTO pl_snapshot_texts + (snapshot_eli, act_eli, http_status, article_count, unit_count, n_chars) + SELECT s.snapshot_eli, s.act_eli, 903, 0, 0, 0 + FROM pl_act_snapshots s + LEFT JOIN pl_snapshot_texts t ON t.snapshot_eli = s.snapshot_eli + WHERE t.snapshot_eli IS NULL AND NOT s.has_html + RETURNING 1;""") + print(f"marked no-html snapshots: {n.strip().splitlines()[-1] if n else '?'}", + flush=True) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--raw", default=RAW) + ap.add_argument("--mark-no-html", action="store_true") + ap.add_argument("--reparse", action="store_true", + help="parse from the on-disk cache only, no network") + a = ap.parse_args() + + globals()["RAW"] = a.raw + os.makedirs(a.raw, exist_ok=True) + + if a.mark_no_html: + mark_no_html() + return 0 + + done = 0 + while True: + n = BATCH if not a.limit else min(BATCH, a.limit - done) + if n <= 0: + break + work = list(plprod.rows_of( + "SELECT s.act_eli, s.snapshot_eli, " + " CASE WHEN s.snapshot_kind='jednolity' THEN 't' ELSE 'f' END " + "FROM pl_act_snapshots s " + "LEFT JOIN pl_snapshot_texts t ON t.snapshot_eli = s.snapshot_eli " + "WHERE t.snapshot_eli IS NULL AND s.has_html " + f"ORDER BY s.act_eli, s.seq LIMIT {n}")) + if not work: + log("worklist empty, done") + break + + arts, units, txts = [], [], [] + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + for aa, uu, tt in pool.map(lambda r: handle(r, a.reparse), work): + arts += aa + units += uu + txts += tt + + # Text rows last: they are what the anti-join keys on, so writing them + # before their articles would let a crash in between leave a snapshot + # marked done with no articles. + if arts: + plprod.copy_rows("pl_act_articles", ART_COLS, arts) + if units: + plprod.copy_rows("pl_act_units", UNIT_COLS, units) + if txts: + plprod.copy_rows("pl_snapshot_texts", TXT_COLS, txts) + + done += len(work) + log(f"batch of {len(work)}") + if a.limit and done >= a.limit: + break + + log("finished") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pl/pin_baseline.py b/scripts/pl/pin_baseline.py new file mode 100644 index 00000000..10c91c19 --- /dev/null +++ b/scripts/pl/pin_baseline.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Stage 0: pin the Sejm ELI corpus baseline before harvesting anything. + +Fetches the two publisher records and all ~197 year listings, stores them +verbatim, and writes baseline.json - the file every later audit asserts against. + +Why this exists as its own step. The corpus moves: /eli/changes/acts reported +376 changed acts in a two-week window, so "we have N acts" is only meaningful +against a pinned N with a timestamp. And the meta-lesson the Ukrainian corpus +paid for is in scripts/legislation/full-corpus/README.md: aggregates cannot tell +"absent upstream" from "we asked wrong". So this also re-measures the landmark +acts whose answers we already know, and fails loudly if any of them moved. + +A year listing is a full metadata dump - GET /eli/acts/DU/2020 returns +count == totalCount == 2463 with 15 fields per item - so the whole register is +enumerable in 197 requests. Act details are NOT fetched here; that is Stage 1. + +Runs anywhere with network. Intended host is local.lex. + + python3 scripts/pl/pin_baseline.py + python3 scripts/pl/pin_baseline.py --out /data/pl_eli/baseline --landmarks-only +""" +import argparse +import json +import os +import subprocess +import sys +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +API = "https://api.sejm.gov.pl/eli" +PUBLISHERS = ("DU", "MP") + +# Measured 2026-08-14: details sustained 11.5 req/s at 5 workers and text.html +# 4.1 req/s at 4, with no 429 across ~700 requests. 4 workers with a 250 ms +# per-worker floor is ~6 req/s aggregate, a deliberate margin rather than a +# limit the source imposed. +WORKERS = int(os.environ.get("WORKERS", "4")) +RATE_MS = int(os.environ.get("RATE_MS", "250")) +TRIES = int(os.environ.get("TRIES", "4")) +TIMEOUT = int(os.environ.get("TIMEOUT", "90")) + +# Every value here was fetched live on 2026-08-14. They are equalities, not +# guidance: a mismatch means either the source changed the act - in which case +# record the new value and say so - or our understanding of the API is wrong. +# Konstytucja RP is the deliberate negative case: it has no HTML, /struct +# returns 404 and text.html returns 200 with zero bytes, so any pipeline that +# assumes "act => HTML" loses the single most recognisable act in the corpus +# without noticing. +LANDMARKS = [ + # eli, label, struct_arti, text_html, struct_ok + ("DU/1964/93", "Kodeks cywilny", 1088, True, True), + ("DU/1997/553", "Kodeks karny", 363, True, True), + ("DU/1997/555", "Kodeks postepowania karnego", 682, True, True), + ("DU/1964/296", "Kodeks postepowania cywilnego",1153, True, True), + ("DU/1974/141", "Kodeks pracy", 305, True, True), + ("DU/1960/168", "Kodeks postepowania adm.", 196, True, True), + ("DU/2020/1320", "KP tekst jednolity 2020", 494, True, True), + ("DU/2023/1610", "KC tekst jednolity 2023", 1295, True, True), + ("DU/1997/483", "Konstytucja RP (PDF only)", 0, False, False), +] + +_last_call = [0.0] + + +def throttle(): + gap = RATE_MS / 1000.0 / max(WORKERS, 1) + now = time.time() + wait = _last_call[0] + gap - now + if wait > 0: + time.sleep(wait) + _last_call[0] = time.time() + + +def fetch(path): + """Return (body_bytes, http_code). curl, not urllib: urllib raises + CERTIFICATE_VERIFY_FAILED against this host in some environments while curl + succeeds, the same class of difference scripts/nl/harvest_bwb_texts.py + documents for KOOP.""" + url = f"{API}/{path}" + for attempt in range(TRIES): + throttle() + r = subprocess.run( + ["curl", "-s", "-m", str(TIMEOUT), "-w", "\n%{http_code}", url], + capture_output=True) + if r.returncode == 0 and r.stdout: + body, _, code = r.stdout.rpartition(b"\n") + try: + code = int(code) + except ValueError: + code = 0 + # A 200 with an empty body is a permanent property of this API for + # acts without HTML, not a transient failure - reproduced on three + # symbol forms and two acts. Do not retry it. + if code in (200, 404): + return body, code + time.sleep(2 * (attempt + 1)) + return None, 599 + + +def fetch_json(path): + body, code = fetch(path) + if code != 200 or not body: + return None, code + try: + return json.loads(body), code + except json.JSONDecodeError: + return None, 901 + + +def year_listing(pub, year): + """One year of one publisher. limit=500 is below the year sizes we see + (max ~2500), but the API returns the whole year regardless: DU/2020 answered + count == totalCount == 2463. The assertion below is what proves that, per + year, rather than assuming it.""" + d, code = fetch_json(f"acts/{pub}/{year}?limit=5000") + if d is None: + return year, None, code, "fetch_failed" + total, got = d.get("totalCount"), len(d.get("items") or []) + status = "ok" if total == got else "TRUNCATED" + return year, d, code, status + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", default=os.environ.get("OUT", "/data/pl_eli/baseline")) + ap.add_argument("--landmarks-only", action="store_true") + args = ap.parse_args() + + os.makedirs(args.out, exist_ok=True) + started = time.time() + report = {"pinned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "api": API, + "publishers": {}, "landmarks": [], "problems": []} + + if not args.landmarks_only: + for pub in PUBLISHERS: + meta, code = fetch_json(f"acts/{pub}") + if meta is None: + report["problems"].append(f"publisher {pub}: HTTP {code}") + continue + # Publisher record is {actsCount, code, name, shortName, years:[int]}. + # DU reports actsCount 97681 over 105 years, MP 66532 over 92. + years = list(meta.get("years") or []) + if not years: + report["problems"].append(f"publisher {pub}: no years listed") + continue + with open(os.path.join(args.out, f"{pub}_publisher.json"), "w") as f: + json.dump(meta, f, ensure_ascii=False) + + per_year, html_by_year, type_counts = {}, {}, Counter() + total_acts = total_html = 0 + + with ThreadPoolExecutor(max_workers=WORKERS) as pool: + for year, d, code, status in pool.map( + lambda y: year_listing(pub, y), years): + if d is None: + report["problems"].append(f"{pub}/{year}: HTTP {code}") + continue + if status == "TRUNCATED": + report["problems"].append( + f"{pub}/{year}: listing truncated, " + f"totalCount={d.get('totalCount')} items={len(d.get('items') or [])}") + with open(os.path.join(args.out, f"{pub}_{year}.json"), "w") as f: + json.dump(d, f, ensure_ascii=False) + items = d.get("items") or [] + n_html = sum(1 for i in items if i.get("textHTML")) + per_year[year] = len(items) + html_by_year[year] = n_html + total_acts += len(items) + total_html += n_html + type_counts.update(i.get("type") for i in items) + print(f" {pub}/{year}: {len(items):>5} acts, {n_html:>5} html", flush=True) + + # The publisher record states its own total. If the year listings do + # not add up to it, the enumeration is incomplete and every later + # coverage percentage would be computed against the wrong + # denominator - the exact way a corpus reports 99% and is wrong. + declared = meta.get("actsCount") + if declared is not None and declared != total_acts: + report["problems"].append( + f"{pub}: actsCount={declared} but year listings sum to {total_acts} " + f"(delta {total_acts - declared})") + + report["publishers"][pub] = { + "declared_acts_count": declared, + "total_acts": total_acts, + "total_html": total_html, + "html_pct": round(100.0 * total_html / total_acts, 2) if total_acts else 0.0, + "years": len(per_year), + "acts_by_year": per_year, + "html_by_year": html_by_year, + "top_types": type_counts.most_common(20), + } + print(f"{pub}: {total_acts} acts, {total_html} with HTML " + f"({100.0*total_html/max(total_acts,1):.1f}%)", flush=True) + + # Landmarks. Measured, then compared - never asserted from memory. + for eli, label, want_arti, want_html, want_struct in LANDMARKS: + meta, code = fetch_json(f"acts/{eli}") + row = {"eli": eli, "label": label, "http": code} + if meta is None: + row["result"] = "FAIL: detail unavailable" + report["problems"].append(f"landmark {eli}: detail HTTP {code}") + report["landmarks"].append(row) + continue + + got_html = bool(meta.get("textHTML")) + struct, s_code = fetch_json(f"acts/{eli}/struct") + got_struct = struct is not None + + n_arti = 0 + if got_struct: + def walk(node): + nonlocal n_arti + if node.get("type") == "arti": + n_arti += 1 + for c in node.get("children") or []: + walk(c) + nodes = struct if isinstance(struct, list) else \ + (struct.get("children") or [struct]) + for n in nodes: + walk(n) + + row.update({"title": meta.get("title"), "type": meta.get("type"), + "status": meta.get("status"), + "legal_status_date": meta.get("legalStatusDate"), + "text_html": got_html, "struct_http": s_code, + "struct_arti": n_arti, "expected_arti": want_arti}) + + bad = [] + if got_html != want_html: + bad.append(f"textHTML {got_html} != {want_html}") + if got_struct != want_struct: + bad.append(f"struct present {got_struct} != {want_struct}") + if want_struct and n_arti != want_arti: + bad.append(f"arti {n_arti} != {want_arti}") + row["result"] = "ok" if not bad else "MISMATCH: " + "; ".join(bad) + if bad: + report["problems"].append(f"landmark {eli} ({label}): {row['result']}") + print(f" landmark {eli:<14} {label:<32} {row['result']}", flush=True) + report["landmarks"].append(row) + + path = os.path.join(args.out, "baseline.json") + with open(path, "w") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + print(f"\nwrote {path} in {int(time.time()-started)}s") + if report["problems"]: + print(f"\n{len(report['problems'])} problem(s):") + for p in report["problems"][:40]: + print(f" - {p}") + # A landmark mismatch must stop the pipeline, not colour a log line. + return 1 + print("no problems: baseline pinned, landmarks all match") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pl/pl_article_parser.py b/scripts/pl/pl_article_parser.py new file mode 100644 index 00000000..684fe5c5 --- /dev/null +++ b/scripts/pl/pl_article_parser.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Split a Polish act's text.html into articles, guided by its /struct tree. + +No database, no network - so it can be tested on fixtures and re-run over cached +HTML when the extraction rules change, without refetching anything. + +Why struct-guided rather than regex-guided. The Ukrainian corpus had to find +articles by matching headings, because zakon.rada published no structure; that +forced the monotonicity heuristic in scripts/legislation/full-corpus/rebuild_articles.py +(article numbers must ascend, MONO_MIN=0.9) as a proxy for "this is a code, not +an amending act quoting one". Poland publishes the structure, and every unit in +the HTML carries id="{struct node id}" data-id="{symbol}" - byte-identical to +the struct tree. So the proxy is replaced by an identity: we extract exactly the +articles struct declares, and any shortfall is an error rather than a ratio. + +That is measurably stronger. On DU/2020/1320 (Kodeks pracy, tekst jednolity +2020) struct declares 494 articles, all under the "Załącznik" part. The HTML +carries 497
anchors: the three extras are +pass_2-pint_1-arti_5, pass_2-pint_1-arti_6 and pass_2-pint_2-arti_86, articles +QUOTED inside the obwieszczenie's own passages. A regex over that HTML returns +497 articles for a 494-article code and no threshold would catch it. + +Verified against fixtures on 2026-08-14: + DU/1974/141 struct 305 arti, DOM 305 anchors, 1:1, ids all distinct + DU/2020/1320 struct 494 arti in annex, DOM 497 anchors, 3 out of scope + DU/1964/93 2,290 struct nodes / 2,289 distinct ids - ids DO repeat +""" +import json +import re +import unicodedata + +import lxml.html + +# Verdict codes, shared with pl_snapshot_texts.http_status. Above the HTTP range +# on purpose, the npa.edition idiom: one column answers "can this row be used" +# for both transport and content failures. +OK = 200 +NO_STRUCT = 902 # act declares textHTML but /struct 404'd +ARTICLE_SHORTFALL = 904 # struct declared N in scope, we produced fewer +LABEL_MISMATCH = 905 # DOM heading disagrees with the struct symbol + +# Polish article numbering has three levels and one special case, all of which +# occur inside a single act (Kodeks pracy, tekst jednolity DU/2020/1320): +# arti_415 Art. 415. -> '415' +# arti_304_4 Art. 3044. -> '304^4' (superscript) +# arti_18_3_a Art. 183a. -> '18^3a' (superscript + letter) +# arti_266-280 Art. 266-280. -> '266-280' (a repealed range as one node) +# The letter-suffixed form accounts for 24 of the 494 articles in that act, so +# rejecting it is not a rounding error. +_SYMBOL_RE = re.compile(r"^arti_(\d+)([a-zA-Z]*)(?:_(\d+))?(?:_([a-zA-Z]+))?$") +_RANGE_SYMBOL_RE = re.compile(r"^arti_(\d+)-(\d+)$") +# The rendered heading after flattening, e.g. "Art. 18^3^a." or "Art. 266-280." +_HEAD_RE = re.compile( + r"Art\.\s*(\d+)(?:-(\d+))?([a-zA-Z]*)(?:\^(\d+))?(?:\^([a-zA-Z]+))?", re.IGNORECASE) +_ANNEX_RE = re.compile(r"^\s*Za[łl][aą]cznik", re.IGNORECASE) + +MIN_ARTICLE_CHARS = 10 +SUBSTANCE_RATIO = 0.95 + + +def _clean(s): + """Collapse whitespace without losing paragraph boundaries. Same shape as + clean() in scripts/nl/harvest_bwb_texts.py, which is the house style.""" + s = s.replace(" ", " ") + s = re.sub(r"[ \t]+", " ", s) + s = re.sub(r" *\n *", "\n", s) + s = re.sub(r"\n{3,}", "\n\n", s) + return s.strip() + + +def art_no_from_symbol(symbol): + """'arti_304_4' -> ('304^4', 304, 4). Returns (None, None, None) if the + symbol is not an addressable article symbol (struct emits `none_` + placeholders that are not articles). + + The number is taken from the symbol and never from the struct node's title, + because the title is lossy: struct renders article 304 superscript 4 as the + ambiguous string 'Art. 304_4.', which cannot be told apart from a + hypothetical article '304_4'. The symbol carries the same '304_4' but with + known semantics, and the DOM carries the unambiguous rendering + 'Art. 3044.' which V2 checks it against. + """ + symbol = symbol or "" + + m = _RANGE_SYMBOL_RE.match(symbol) + if m: + # A repealed span of articles published as a single unit, e.g. + # "Art. 266-280." in the twelfth division of the Kodeks pracy. Sorts on + # the first number so it lands in the right place in the chain. + return f"{m.group(1)}-{m.group(2)}", int(m.group(1)), None + + m = _SYMBOL_RE.match(symbol) + if not m: + return None, None, None + base, letter, sup, sup_letter = (m.group(1), m.group(2) or "", + m.group(3), m.group(4) or "") + canon = f"{base}{letter}" + if sup: + canon += f"^{sup}{sup_letter}" + elif sup_letter: + canon += sup_letter + return canon, int(base), int(sup) if sup else None + + +def art_no_from_heading(display): + """'Art. 18^3^a.' -> '18^3a'. None when the heading carries no number. + + Deliberately shares its output form with art_no_from_symbol: V2 compares the + two, so if the canonical forms were built in two places they could drift and + the check would compare a parser bug against itself. + """ + m = _HEAD_RE.search(display or "") + if not m: + return None + base, rng_end, letter, sup, sup_letter = m.groups() + if rng_end: + return f"{base}-{rng_end}" + canon = f"{base}{letter or ''}" + if sup: + canon += f"^{sup}{sup_letter or ''}" + elif sup_letter: + canon += sup_letter + return canon + + +def repair_struct_json(raw): + """Parse a /struct payload, repairing the source's malformed JSON. + + ISAP's /struct serialiser has two independent defects, both appearing in + title fields, and either one makes the entire payload unparseable - so the + act gets no struct, and therefore no articles at all. + + Defect 1, unescaped ASCII double quotes. Polish typography opens a + quotation with the low quote and the source often closes it with a straight + ", which terminates the JSON string early: + + "title" : "...panstwowego ,,Polskie Koleje Panstwowe"1)", + + DU/2024/561 is 76 KB and 140 articles behind one such quotation mark. + + Defect 2, literal control characters. Long table captions are wrapped + across physical lines and the newline is emitted raw inside the string, + which JSON forbids (DU/2020/2075). + + The repair walks the payload, escaping a quote inside a string that is not + followed by a structural delimiter, and escaping raw newline/CR/tab while + inside a string. Outside a string those characters are legal whitespace, + hence the in_string guard on both rules. + + Returns (parsed, repaired_bool) and still raises json.JSONDecodeError if + the payload will not parse, so a genuinely broken one stays an honest + failure rather than a silent empty result. + """ + if isinstance(raw, (bytes, bytearray)): + raw = raw.decode("utf-8", errors="replace") + try: + return json.loads(raw), False + except json.JSONDecodeError: + pass + + out = [] + in_string = False + escaped = False + for i, ch in enumerate(raw): + if escaped: + out.append(ch) + escaped = False + continue + if ch == "\\": + # Defect 3: a trailing backslash in the content, unescaped. A title + # ending "Zalacznik - WZOR\" leaves \" looking like an escaped + # quote, so the string never closes. If what follows the quote is a + # structural delimiter, the backslash is literal content and the + # quote really is the terminator (DU/2018/428). + if in_string and i + 1 < len(raw) and raw[i + 1] == '"': + after = "" + for c in raw[i + 2:]: + if not c.isspace(): + after = c + break + if after in (",", ":", "}", "]", ""): + out.append("\\\\") + continue + out.append(ch) + escaped = True + continue + if in_string and ch in "\n\r\t": + # Defect 2: literal control characters inside a string value. JSON + # forbids them; the source wraps long table captions across lines + # and emits the newline raw. Outside a string they are legal + # whitespace, hence the in_string guard. + out.append({"\n": "\\n", "\r": "\\r", "\t": "\\t"}[ch]) + continue + if ch == '"': + if not in_string: + in_string = True + out.append(ch) + continue + # Closing quote only if the next non-space character is one that can + # legally follow a string. Anything else means the source left a + # quote inside the value. + nxt = "" + for c in raw[i + 1:]: + if not c.isspace(): + nxt = c + break + if nxt in (",", ":", "}", "]", ""): + in_string = False + out.append(ch) + else: + out.append('\\"') + continue + out.append(ch) + + return json.loads("".join(out)), True + + +def _iter_struct(struct): + """Preorder walk. Yields (node, depth, parent_ord, ord, top_id, in_annex).""" + roots = struct if isinstance(struct, list) else (struct.get("children") or [struct]) + counter = [0] + + def walk(node, depth, parent_ord, top_id, in_annex): + ord_i = counter[0] + counter[0] += 1 + yield node, depth, parent_ord, ord_i, top_id, in_annex + for child in node.get("children") or []: + yield from walk(child, depth + 1, ord_i, top_id, in_annex) + + for root in roots: + in_annex = bool(_ANNEX_RE.match(root.get("title") or "")) + yield from walk(root, 0, None, root.get("id"), in_annex) + + +def strip_glosses(tree): + """Remove the editorial footnote apparatus, and return how many were dropped. + + ISAP renders a footnote as + + 3)W brzmieniu ustalonym przez ... + + inline, inside the very heading or paragraph it annotates. Two things go + wrong if it is left in place. + + First, the tooltip prose becomes part of the provision: 21,332 characters + across 44 glosses in DU/2020/1320 (1.6% of the body), and a reader or a + retrieval index cannot tell the legislator's words from the publisher's + note about which amendment changed them. + + Second, the 3) marker sits immediately after the article number, + so flattening it turns "Art. 47" into "Art. 47^6" and the article appears to + be a superscript article that disagrees with its own symbol. That produced + two false LABEL_MISMATCH verdicts on DU/2020/1320 and six on DU/2023/1610 - + i.e. the check meant to catch mis-pairing was firing on formatting. + + Article-number superscripts are distinguishable by markup, not by content: + they are bare 3 directly inside the heading's , never wrapped + in a gloss-link. + """ + dropped = 0 + for a in tree.xpath('//a[contains(@class, "gloss-link")]'): + parent = a.getparent() + if parent is None: + continue + # Keep the tail: it is the text that follows the footnote marker and + # belongs to the provision. + tail = a.tail or "" + if tail: + prev = a.getprevious() + if prev is not None: + prev.tail = (prev.tail or "") + tail + else: + parent.text = (parent.text or "") + tail + parent.remove(a) + dropped += 1 + # Any leftover tooltip spans not wrapped in a gloss-link. + for span in tree.xpath('//span[contains(@class, "tooltip-text")]'): + p = span.getparent() + if p is not None: + p.remove(span) + dropped += 1 + return dropped + + +def _flatten_sup(tree): + """Rewrite 4 as the literal '^4' in the text stream, so that + article 304 superscript 4 survives itertext() as 'Art. 304^4' instead of + collapsing to the unreadable and ambiguous 'Art. 3044'. + + Must run AFTER strip_glosses, or footnote markers get the same treatment. + """ + for sup in tree.iter(): + if sup.tag and str(sup.tag).lower() == "sup": + inner = "".join(sup.itertext()).strip() + sup.text = f"^{inner}" if inner else "" + for child in list(sup): + sup.remove(child) + + +def _text_with_offsets(el, want_ids, skip=()): + """Text of an element, plus {id: (char_from, char_to)} for the descendants + whose id is in want_ids. + + Offsets rather than nested copies: 'art. 415 § 1' is then a slice of the + article's text, and no provision is ever stored twice. + """ + parts = [] + spans = {} + length = [0] + + def emit(s): + if s: + parts.append(s) + length[0] += len(s) + + def walk(node): + if node in skip: + # The article's own "Art. 415." heading: already carried by + # art_display, so repeating it inside text would store the label + # twice and prefix every retrieved provision with its own number. + # Nested headings (a paragraph's "§ 1.") are NOT skipped - those + # are structure inside the provision. The caller emits node.tail + # after this returns, so it must not be emitted here as well. + return + node_id = node.get("id") + tracked = node_id in want_ids if node_id else False + start = length[0] + emit(node.text) + for child in node: + walk(child) + emit(child.tail) + tag = (str(node.tag) or "").lower() + if tag in ("div", "p", "h1", "h2", "h3", "h4", "li", "br", "tr"): + emit("\n") + if tracked: + spans[node_id] = (start, length[0]) + return + + walk(el) + raw = "".join(parts) + cleaned = _clean(raw) + # _clean shifts offsets. Rather than track the rewrite, rescale + # proportionally only when nothing moved; otherwise drop the spans for this + # article. A wrong offset silently returns the wrong provision, which is + # worse than returning none, so the honest failure is to have no span. + if len(raw) != len(cleaned): + spans = {} + return cleaned, spans + + +class ParseResult: + def __init__(self): + self.articles = [] # dicts, see _make_article + self.units = [] # dicts mirroring pl_act_units + self.verdict = OK + self.annex_part_id = None + self.struct_articles = 0 + self.dom_anchors = 0 + self.label_mismatches = 0 + self.nonmonotonic = 0 + self.glosses_dropped = 0 + self.notes = [] + + +def parse(struct, html_bytes, is_consolidation=False): + """Return a ParseResult. Never raises on content; verdicts carry the failure.""" + res = ParseResult() + + if struct is None: + res.verdict = NO_STRUCT + res.notes.append("no struct") + return res + + nodes = list(_iter_struct(struct)) + + # Scope. On a consolidating obwieszczenie the law IS the annex: part_1 is + # "Treść obwieszczenia" (the Marshal's announcement) and part_2 is + # "Załącznik - Tekst jednolity ustawy ...". Only the annex is the act. + if is_consolidation: + annex_tops = {n[4] for n in nodes if n[5]} + if not annex_tops: + res.verdict = ARTICLE_SHORTFALL + res.notes.append("consolidation with no Załącznik part") + return res + res.annex_part_id = sorted(annex_tops)[0] + in_scope = [n for n in nodes if n[5]] + else: + in_scope = nodes + + arti_nodes = [n for n in in_scope if n[0].get("type") == "arti"] + res.struct_articles = len(arti_nodes) + + # Force UTF-8. The documents declare + # + # - the charset is stated twice in one attribute - and lxml's sniffing gives + # up on that and falls back to latin-1. The damage is not only mojibake in + # the text: struct ids contain Polish letters (bran_piąty-chpt_I-arti_114), + # so a mis-decoded id stops matching the struct tree and the article is + # silently dropped. On Kodeks pracy that lost 134 of 305 articles while + # every anchor count still looked right. + doc = lxml.html.document_fromstring( + html_bytes, parser=lxml.html.HTMLParser(encoding="utf-8")) + res.glosses_dropped = strip_glosses(doc) + _flatten_sup(doc) + + # ids repeat (DU/1964/93 lists book_trzecia-titl_XI-bran_I-arti_538 twice), + # so index to a LIST and consume the k-th occurrence for the k-th struct + # node. A dict keyed by id would silently drop one of the two. + dom_by_id = {} + for el in doc.iter("div"): + cls = el.get("class") or "" + if "unit" not in cls: + continue + el_id = el.get("id") + if el_id: + dom_by_id.setdefault(el_id, []).append(el) + res.dom_anchors = sum(1 for el in doc.iter("div") + if "unit_arti" in (el.get("class") or "")) + + consumed = {} + ord_i = 0 + prev_sort = (-1, -1) + unit_rows = [] + art_by_struct_id = {} + + for node, depth, parent_ord, node_ord, top_id, in_annex in in_scope: + unit_rows.append({ + "ord": node_ord, "parent_ord": parent_ord, "depth": depth, + "struct_id": node.get("id"), "symbol": node.get("symbol"), + "unit_type": node.get("type"), "name": node.get("name"), + "title": node.get("title"), "in_annex": in_annex, + "article_ord": None, "char_from": None, "char_to": None, + }) + + for node, depth, parent_ord, node_ord, top_id, in_annex in arti_nodes: + struct_id = node.get("id") + symbol = node.get("symbol") + art_no, sort1, sort2 = art_no_from_symbol(symbol) + if art_no is None: + # Not an addressable article (struct emits `none_` placeholders). + continue + + k = consumed.get(struct_id, 0) + candidates = dom_by_id.get(struct_id) or [] + if k >= len(candidates): + continue + consumed[struct_id] = k + 1 + el = candidates[k] + + # Descendants of this article that struct also knows about, so their + # offsets can be recorded while the text is built. + want = {u["struct_id"] for u in unit_rows + if u["struct_id"] and u["struct_id"].startswith(struct_id + "-")} + + # The article's own heading is its DIRECT child h3, not the first h3 in + # the subtree - a paragraph's "§ 1." heading is also an h3, and + # el.find(".//h3") would return whichever comes first in document order. + head = next((c for c in el if str(c.tag).lower() == "h3"), None) + display = _clean("".join(head.itertext())) if head is not None else "" + text, spans = _text_with_offsets(el, want, skip={head} if head is not None else ()) + + # V2: the number the document renders must equal the number the symbol + # encodes. This is what catches an off-by-one in the occurrence pairing + # above, which no aggregate check would show. + head_no = art_no_from_heading(display) + if head_no is not None and head_no != art_no: + res.label_mismatches += 1 + + # V3: residual, reported and never used to discard a document - but it + # earns its place. In DU/1964/93 the article at document position 536 is + # labelled "Art. 538." by BOTH struct and the DOM heading, while its + # text is the real article 536 ("Cenę można określić przez wskazanie + # podstaw do jej ustalenia"), and the real 538 follows at position 538. + # That is a defect in the published source. V2 structurally cannot see + # it, because V2 compares struct against the DOM and here the two agree + # with each other and are both wrong. Only the ordering shows it. So a + # non-zero nonmonotonic count is a finding to be surfaced by the audit, + # not a number expected to be zero. + cur = (sort1, sort2 if sort2 is not None else -1) + if cur < prev_sort: + res.nonmonotonic += 1 + prev_sort = cur + + ord_i += 1 + art_by_struct_id[struct_id] = ord_i + res.articles.append({ + "ord": ord_i, "symbol": symbol, "struct_id": struct_id, + "art_no": art_no, "art_display": display or f"Art. {art_no}.", + "art_sort_1": sort1, "art_sort_2": sort2, + "art_title": node.get("title"), + "text": text, "n_chars": len(text), "spans": spans, + }) + + # Attach sub-unit offsets to their article. + art_ord_by_prefix = sorted(art_by_struct_id.items(), key=lambda kv: -len(kv[0])) + span_lookup = {} + for a in res.articles: + for uid, (cf, ct) in a["spans"].items(): + span_lookup[uid] = (a["ord"], cf, ct) + for u in unit_rows: + sid = u["struct_id"] + if not sid: + continue + if sid in span_lookup: + u["article_ord"], u["char_from"], u["char_to"] = span_lookup[sid] + else: + for prefix, a_ord in art_ord_by_prefix: + if sid == prefix or sid.startswith(prefix + "-"): + u["article_ord"] = a_ord + break + res.units = unit_rows + + # V1: exact coverage. Not a ratio and not a threshold - struct states how + # many articles are in scope, and anything less means a truncated download, + # a renamed anchor or a parse that stopped mid-document. + addressable = sum(1 for n in arti_nodes + if art_no_from_symbol(n[0].get("symbol"))[0] is not None) + if len(res.articles) != addressable: + res.verdict = ARTICLE_SHORTFALL + res.notes.append(f"extracted {len(res.articles)} of {addressable} in-scope articles") + return res + + # V4: substance. Catches a stylesheet-only or JS-shell render that still + # produced the right number of empty anchors. + if res.articles: + substantial = sum(1 for a in res.articles if a["n_chars"] >= MIN_ARTICLE_CHARS) + if substantial / len(res.articles) < SUBSTANCE_RATIO: + res.verdict = ARTICLE_SHORTFALL + res.notes.append( + f"only {substantial}/{len(res.articles)} articles have " + f">={MIN_ARTICLE_CHARS} chars") + return res + + if res.label_mismatches: + res.verdict = LABEL_MISMATCH + res.notes.append(f"{res.label_mismatches} DOM/symbol label mismatches") + + return res diff --git a/scripts/pl/plprod.py b/scripts/pl/plprod.py new file mode 100644 index 00000000..c8c08eb2 --- /dev/null +++ b/scripts/pl/plprod.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Postgres access for the Polish corpus scripts: read worklists, COPY rows in. + +Prod Postgres has no public listener, so everything goes through +ssh -> docker exec -i secondlayer-postgres-prod psql, the same route +services/opendata-importers/shared/prod_writer.py and scripts/nl/harvest_bwb_texts.py +take. Set PL_SSH_HOST="" when running ON the prod host to drop the ssh hop. + +Why this is not prod_writer.copy_into. That helper's _escape_copy maps newline +to a space: + + .replace("\\t", " ").replace("\\n", " ").replace("\\r", " ") + +which is right for the metadata rows it was written for and wrong for statute +text: a Polish article is a stack of numbered paragraphs, and flattening it to +one line destroys the structure that makes "art. 415 § 1" addressable at all. +esc() below emits a literal backslash-n instead, which COPY ... FORMAT text +decodes back into a real newline - the escaping scripts/nl/harvest_bwb_texts.py +already uses for the same reason. +""" +import os +import shlex +import subprocess + +SSH_HOST = os.environ.get("PL_SSH_HOST", "prod") +CONTAINER = os.environ.get("PG_CONTAINER", "secondlayer-postgres-prod") +DB_USER = os.environ.get("PG_USER", "secondlayer") +DB_NAME = os.environ.get("PG_DB", "secondlayer_prod") +PSQL_TIMEOUT = int(os.environ.get("PSQL_TIMEOUT", "1800")) + + +def _argv(extra=()): + """argv for psql, optionally through ssh. + + extra must be passed in here rather than appended by the caller: over ssh + the whole remote command is one string handed to a remote shell, so an + argument containing spaces or parentheses - every "-c COPY t (a, b) FROM + STDIN" - has to be quoted before it is joined. Appending to the returned + list works locally and produces a remote bash syntax error. + """ + inner = ["docker", "exec", "-i", CONTAINER, + "psql", "-U", DB_USER, "-d", DB_NAME, "-v", "ON_ERROR_STOP=1", + *extra] + if not SSH_HOST: + return inner + return ["ssh", "-o", "ConnectTimeout=20", "-o", "ServerAliveInterval=30", + SSH_HOST, " ".join(shlex.quote(a) for a in inner)] + + +def psql(sql, stdin=None): + r = subprocess.run(_argv(["-c", sql]), input=stdin, + capture_output=True, text=True, timeout=PSQL_TIMEOUT) + if r.returncode != 0: + raise RuntimeError(f"psql failed: {r.stderr.strip()[:600]}") + return r.stdout + + +def esc(v): + r"""One cell for COPY ... WITH (FORMAT text). + + Newlines become the two-character sequence \n, which COPY turns back into a + newline on load - unlike replacing them with spaces, which is lossy. + Booleans are emitted as t/f, which is what COPY expects. + """ + if v is None: + return r"\N" + if isinstance(v, bool): + return "t" if v else "f" + # An empty string is NOT null here, unlike in prod_writer._escape_copy. + # A repealed article legitimately has no body - the Kodeks pracy carries + # "Art. 266-280." as a single unit covering a repealed span, with a heading + # and nothing else - and mapping that to NULL both loses the distinction + # between "repealed" and "we failed to extract it" and violates the NOT NULL + # on pl_act_articles.text. + return (str(v).replace("\\", "\\\\") + .replace("\t", "\\t") + .replace("\n", "\\n") + .replace("\r", "")) + + +def copy_rows(table, columns, rows, batch=2000): + """COPY rows into table. Returns the number of rows sent. + + Straight COPY, no ON CONFLICT: every caller in this corpus writes to a table + whose worklist is an anti-join, so a row is only produced when it is not + already there. Upserting is sync_eli_changes.py's job and it stages + explicitly. + """ + rows = list(rows) + if not rows: + return 0 + col_sql = ", ".join(columns) + for i in range(0, len(rows), batch): + chunk = rows[i:i + batch] + data = "".join("\t".join(esc(c) for c in r) + "\n" for r in chunk) + r = subprocess.run( + _argv(["-c", f"COPY {table} ({col_sql}) FROM STDIN WITH (FORMAT text)"]), + input=data, capture_output=True, text=True, timeout=PSQL_TIMEOUT) + if r.returncode != 0: + raise RuntimeError(f"COPY into {table} failed: {r.stderr.strip()[:600]}") + return len(rows) + + +def upsert_rows(table, columns, rows, pk_columns, prefer_new=True, batch=2000): + """Stage into a TEMP table, then INSERT ... ON CONFLICT DO UPDATE. + + prefer_new uses COALESCE(EXCLUDED.col, table.col) so a re-fetch that + populates a previously-NULL field fills it in without a fetch that returned + less blanking what we already hold. That matters here: the source + retroactively adds legalStatusDate to old obwieszczenia and flips textHTML + true for acts published without HTML. + """ + rows = list(rows) + if not rows: + return 0 + col_sql = ", ".join(columns) + if prefer_new: + sets = ", ".join( + f"{c} = COALESCE(EXCLUDED.{c}, {table}.{c})" + for c in columns if c not in pk_columns) + else: + sets = ", ".join(f"{c} = EXCLUDED.{c}" for c in columns if c not in pk_columns) + + for i in range(0, len(rows), batch): + chunk = rows[i:i + batch] + data = "".join("\t".join(esc(c) for c in r) + "\n" for r in chunk) + # BEGIN/COMMIT is not decoration. psql autocommits each statement, so a + # TEMP ... ON COMMIT DROP table is created and dropped before the COPY + # on the next line can see it ("relation _stage does not exist"). The + # explicit transaction also makes the batch atomic: a bad row aborts the + # whole batch instead of leaving it half applied. + # + # The COPY data follows the script on the same stdin, terminated by \., + # which is how psql scripts feed COPY FROM STDIN. + sql = ( + "BEGIN;\n" + f"CREATE TEMP TABLE _stage (LIKE {table} INCLUDING DEFAULTS) ON COMMIT DROP;\n" + f"COPY _stage ({col_sql}) FROM STDIN WITH (FORMAT text);\n") + tail = ( + f"INSERT INTO {table} ({col_sql}) SELECT {col_sql} FROM _stage\n" + f" ON CONFLICT ({', '.join(pk_columns)}) DO UPDATE SET {sets};\n" + "COMMIT;\n") + r = subprocess.run(_argv(), input=sql + data + "\\.\n" + tail, + capture_output=True, text=True, timeout=PSQL_TIMEOUT) + if r.returncode != 0: + raise RuntimeError(f"upsert into {table} failed: {r.stderr.strip()[:600]}") + return len(rows) + + +def rows_of(sql): + """Run a COPY (...) TO STDOUT and yield tuples of raw strings. + + \\N comes back as None; the two-character \\n sequence is turned back into a + newline, mirroring esc(). + """ + out = psql(f"COPY ({sql}) TO STDOUT WITH (FORMAT text)") + for line in out.split("\n"): + if not line: + continue + cells = [] + for c in line.split("\t"): + cells.append(None if c == r"\N" + else c.replace("\\n", "\n").replace("\\t", "\t") + .replace("\\\\", "\\")) + yield tuple(cells) + + +def scalar(sql): + return psql(f"SELECT ({sql})").strip().splitlines()[2].strip() diff --git a/scripts/pl/test_pl_article_parser.py b/scripts/pl/test_pl_article_parser.py new file mode 100644 index 00000000..d6941e82 --- /dev/null +++ b/scripts/pl/test_pl_article_parser.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Fixture tests for pl_article_parser. + +Fixtures are real API responses cached under FIXTURES (default +/data/pl_eli/fixtures). Fetch them once with: + + python3 scripts/pl/test_pl_article_parser.py --fetch + +Every expected number below was measured against the live API on 2026-08-14 and +is asserted as an equality. If one of them moves, the source changed the act - +record the new value with the date - or the parser regressed. Do not relax an +assertion to make it pass. + +Run: python3 scripts/pl/test_pl_article_parser.py +""" +import argparse +import json +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import pl_article_parser as P # noqa: E402 + +API = "https://api.sejm.gov.pl/eli" +FIXTURES = os.environ.get("FIXTURES", "/data/pl_eli/fixtures") + +CASES = [ + # eli, is_consolidation, articles, struct arti in scope, DOM anchors, + # verdict, non-monotonic count + # + # DU/2020/1320: 497 DOM anchors vs 494 articles is correct. The three extras + # are pass_2-pint_1-arti_5/6 and pass_2-pint_2-arti_86 - articles QUOTED in + # the obwieszczenie's own passages, which struct excludes from the annex. + # + # DU/1964/93 nonmono=1 is a defect in the published source, not in the + # parser: the article at position 536 is labelled "Art. 538." by both struct + # and the DOM, while its text is the real article 536. Pinned so that the + # day it changes - because ISAP fixed it, or because we broke ordering - it + # shows up instead of passing quietly. + ("DU/1974/141", False, 305, 305, 305, P.OK, 0), + ("DU/2020/1320", True, 494, 494, 497, P.OK, 0), + ("DU/1964/93", False, 1088, 1088, 1088, P.OK, 1), + ("DU/2023/1610", True, 1295, 1295, 1302, P.OK, 0), + ("DU/1997/553", False, 363, 363, 363, P.OK, 0), +] + + +def fname(eli, kind): + return os.path.join(FIXTURES, eli.replace("/", "_") + kind) + + +def fetch_all(): + os.makedirs(FIXTURES, exist_ok=True) + for eli, *_ in CASES: + for kind, path in (("/text.html", ".html"), ("/struct", ".struct.json")): + out = fname(eli, path) + if os.path.exists(out) and os.path.getsize(out) > 0: + print(f" have {out}") + continue + subprocess.run(["curl", "-s", "-m", "120", f"{API}/acts/{eli}{kind}", + "-o", out], check=True) + print(f" fetched {out} ({os.path.getsize(out)} bytes)") + # The negative landmark: no HTML at all, so only the detail is stored. + out = fname("DU/1997/483", ".detail.json") + subprocess.run(["curl", "-s", "-m", "60", f"{API}/acts/DU/1997/483", "-o", out], + check=True) + print(f" fetched {out}") + + +def run(): + failures = [] + + for (eli, is_cons, want_articles, want_struct, want_anchors, + want_verdict, want_nonmono) in CASES: + html_p, struct_p = fname(eli, ".html"), fname(eli, ".struct.json") + if not (os.path.exists(html_p) and os.path.exists(struct_p)): + failures.append(f"{eli}: fixtures missing, run with --fetch") + continue + + with open(html_p, "rb") as f: + html = f.read() + with open(struct_p, encoding="utf-8") as f: + struct = json.load(f) + + r = P.parse(struct, html, is_consolidation=is_cons) + checks = [ + ("verdict", r.verdict, want_verdict), + ("articles", len(r.articles), want_articles), + ("struct_articles", r.struct_articles, want_struct), + ("nonmonotonic", r.nonmonotonic, want_nonmono), + # Label agreement is the check that catches mis-pairing of repeated + # struct ids, so it is asserted at zero everywhere rather than only + # folded into the verdict. + ("label_mismatches", r.label_mismatches, 0), + ] + if want_anchors is not None: + checks.append(("dom_anchors", r.dom_anchors, want_anchors)) + + bad = [f"{n}: got {g}, want {w}" for n, g, w in checks if g != w] + status = "ok" if not bad else "FAIL" + print(f"{eli:<14} cons={str(is_cons):<5} arts={len(r.articles):>5} " + f"struct={r.struct_articles:>5} dom={r.dom_anchors:>5} " + f"verdict={r.verdict} labelmm={r.label_mismatches} " + f"nonmono={r.nonmonotonic} {status}") + for b in bad: + print(f" - {b}") + failures.append(f"{eli}: {b}") + for note in r.notes: + print(f" note: {note}") + + # --- content assertions: the number being right does not mean the text is --- + + def articles_of(eli, is_cons): + with open(fname(eli, ".html"), "rb") as f: + html = f.read() + with open(fname(eli, ".struct.json"), encoding="utf-8") as f: + struct = json.load(f) + r = P.parse(struct, html, is_consolidation=is_cons) + return {a["art_no"]: a for a in r.articles}, r + + if os.path.exists(fname("DU/1974/141", ".html")): + kp74, _ = articles_of("DU/1974/141", False) + kp20, r20 = articles_of("DU/2020/1320", True) + + # The base act's text.html is the 1974 ORIGINAL, not a consolidation. + # This is the assertion that would have caught the original plan's wrong + # assumption that text.html serves a current consolidated text. + if "socjalistycznych" not in kp74.get("1", {}).get("text", ""): + failures.append("DU/1974/141 art. 1 lost the 1974 wording") + # ...and the obwieszczenie's text.html is the consolidation. + if "socjalistyczn" in kp20.get("1", {}).get("text", ""): + failures.append("DU/2020/1320 art. 1 still carries the 1974 wording") + print(f"\nKP art.1 1974: {kp74.get('1', {}).get('text', '')[:70]!r}") + print(f"KP art.1 2020: {kp20.get('1', {}).get('text', '')[:70]!r}") + + # Superscript articles survive as addressable numbers. + if "304^4" not in kp20: + failures.append("DU/2020/1320 has no art_no '304^4'") + else: + a = kp20["304^4"] + if (a["art_sort_1"], a["art_sort_2"]) != (304, 4): + failures.append(f"304^4 sorts as {a['art_sort_1']},{a['art_sort_2']}") + print(f"KP 304^4 display={a['art_display']!r} sort=({a['art_sort_1']},{a['art_sort_2']})") + + # The three quoted articles inside the obwieszczenie's own passages must + # NOT have become articles of the code. + quoted = [a for a in r20.articles if a["struct_id"].startswith("pass_")] + if quoted: + failures.append(f"{len(quoted)} articles leaked from obwieszczenie passages") + + if os.path.exists(fname("DU/2023/1610", ".html")): + kc23, _ = articles_of("DU/2023/1610", True) + want = "Kto z winy swej wyrządził drugiemu szkodę, obowiązany jest do jej naprawienia." + got = kc23.get("415", {}).get("text", "").strip() + if got != want: + failures.append(f"KC art. 415 text mismatch: {got[:90]!r}") + print(f"KC 415 (t.j. 2023): {got!r}") + + # The negative landmark. Konstytucja RP publishes no HTML and no struct, so + # the parser must return a verdict rather than an empty success. + r = P.parse(None, b"", is_consolidation=False) + if r.verdict != P.NO_STRUCT: + failures.append(f"no-struct case returned verdict {r.verdict}, want {P.NO_STRUCT}") + else: + print(f"\nno-struct case -> verdict {r.verdict} (Konstytucja RP DU/1997/483 path)") + + print() + if failures: + print(f"{len(failures)} FAILURE(S):") + for f in failures: + print(f" - {f}") + return 1 + print("all fixture tests passed") + return 0 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--fetch", action="store_true", help="download fixtures first") + a = ap.parse_args() + if a.fetch: + fetch_all() + sys.exit(run())