diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index b83a3b61f..90f98dace 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -153,6 +153,17 @@ CREATE TABLE contracts ( created_at TEXT NOT NULL DEFAULT (datetime('now')) ); +-- Joint procurements retain one lead authority on tenders (the only authority that receives spend), +-- while this bridge records every participating authority. ordinal 0 is always the lead. +CREATE TABLE contract_co_authorities ( + contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE, + authority_id TEXT NOT NULL REFERENCES authorities(id), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + PRIMARY KEY (contract_id, authority_id) +); +CREATE INDEX idx_contract_co_authorities_authority + ON contract_co_authorities(authority_id, contract_id); + -- Domain amendment history used to roll current_value/annex_count without staging. Natural-keyed so -- re-imports are idempotent; built by scripts/promote-amendments.sql from the transient amendment feed. CREATE TABLE amendments ( @@ -246,6 +257,15 @@ CREATE TABLE authority_totals ( last_date TEXT ); +-- Joint-procurement participation rollup. joint_contract_value_eur is the total value of the joint +-- procurements the authority took part in; it is informational only and must NEVER be summed into +-- authority_totals.spent_eur or any national/leaderboard total. +CREATE TABLE authority_joint_participation ( + authority_id TEXT PRIMARY KEY REFERENCES authorities(id), + joint_contract_participations INTEGER NOT NULL, + joint_contract_value_eur REAL NOT NULL DEFAULT 0 +); + -- Per CPV division. Sector facet + filter counts on the list pages. CREATE TABLE sector_totals ( division TEXT PRIMARY KEY, -- 2-digit CPV division (joins @sigma/config CPV_SECTORS) diff --git a/packages/db/src/migrations.test.ts b/packages/db/src/migrations.test.ts index 3e26faba7..f0fcb9853 100644 --- a/packages/db/src/migrations.test.ts +++ b/packages/db/src/migrations.test.ts @@ -34,6 +34,19 @@ describe('served migrations', () => { "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='amendments';", ).trim(), ).toBe('1'); + + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='contract_co_authorities';", + ).trim(), + ).toBe('1'); + expect( + sqlite( + dbPath, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='authority_joint_participation';", + ).trim(), + ).toBe('1'); expect( sqlite( dbPath, diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index 8470d55df..c64b3e38c 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -11,6 +11,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); function sqlite(dbPath: string, sql: string): string { @@ -352,6 +353,140 @@ function seedTwoCollidingBidders(dbPath: string): void { } } +function seedJointAuthorityFixture(dbPath: string, refresh: boolean): void { + if (refresh) { + sqlite( + dbPath, + `INSERT INTO authorities (id, name, bulstat, type) + VALUES ('auth:222222222', 'Second Authority', '222222222', 'public'); + INSERT INTO tenders + (id, source_id, title, authority_id, currency, procedure_type, status) + VALUES + ('t:00080-2023-0001', '00080-2023-0001', 'Historical prefix observation', + 'auth:222222222', 'EUR', 'open', 'published');`, + ); + } + + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject, + estimated_value, currency, authority_name, authority_eik, authority_type, lot_id, published_at) + VALUES + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2023-0001', + 'TENDER-PREFIX-HISTORY', 'open', 'Historical prefix observation', 1000, 'EUR', + 'Second Authority', '222222222', 'public', NULL, '2023-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '99999-2023-0001', + 'TENDER-NAME-HISTORY', 'open', 'Historical name observation', 1000, 'EUR', + 'First Authority', '111111111', 'public', NULL, '2023-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2024-0030', + 'TENDER-JOINT-PREFIX', 'open', 'Prefix-led joint tender', 1000, 'EUR', + 'First Authority', '111111111;222222222', 'public', NULL, '2024-01-01'), + ('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', 'UNRESOLVED-2024-0001', + 'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR', + 'Third Authority', '333333333;444444444', 'public', NULL, '2024-01-01'); + + INSERT INTO raw_contracts + (source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number, + published_at, unp, tender_ext_id, procedure_type, procurement_subject, estimated_value, + procurement_currency, authority_name, authority_eik, authority_type, contract_number, + contract_date, signing_value, currency, contract_subject, awarded_to_group, + contractor_eik, contractor_name, eu_funded, bids_received) + VALUES + ('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0, + 'DOC-JOINT-PREFIX', '2024-01-01', '00080-2024-0030', 'TENDER-JOINT-PREFIX', 'open', + 'Prefix-led joint tender', 1000, 'EUR', 'First Authority', '111111111;222222222', + 'public', 'CONTRACT-JOINT-PREFIX', '2024-01-02', 100, 'EUR', 'Prefix-led contract', 0, + '777777777', 'Joint Bidder', 0, 1), + ('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0, + 'DOC-JOINT-FALLBACK', '2024-01-01', 'UNRESOLVED-2024-0001', + 'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR', + 'Third Authority', '333333333;444444444', 'public', 'CONTRACT-JOINT-FALLBACK', + '2024-01-02', 200, 'EUR', 'Fallback contract', 0, '777777777', 'Joint Bidder', 0, 1);`, + ); +} + +describe('joint-procurement authority attribution', () => { + it.each([ + ['full normalize', false], + ['slice refresh', true], + ] as const)( + 'uses the УНП prefix and keeps participation value non-summable on %s', + (_label, refresh) => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-joint-authorities-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + initWorkDb(dbPath); + seedJointAuthorityFixture(dbPath, refresh); + readScript(dbPath, refresh ? refreshSlicePath : normalizePath); + if (!refresh) readScript(dbPath, precomputePath); + + expect( + sqliteJson<{ source_id: string; authority_id: string }>( + dbPath, + `SELECT source_id, authority_id FROM tenders + WHERE source_id IN ('00080-2024-0030', 'UNRESOLVED-2024-0001') + ORDER BY source_id`, + ), + ).toEqual([ + { source_id: '00080-2024-0030', authority_id: 'auth:222222222' }, + { source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333' }, + ]); + + expect( + sqliteJson<{ source_id: string; authority_id: string; ordinal: number }>( + dbPath, + `SELECT t.source_id, cca.authority_id, cca.ordinal + FROM contract_co_authorities cca + JOIN contracts c ON c.id = cca.contract_id + JOIN tenders t ON t.id = c.tender_id + WHERE cca.ordinal = 0 + ORDER BY t.source_id`, + ), + ).toEqual([ + { source_id: '00080-2024-0030', authority_id: 'auth:222222222', ordinal: 0 }, + { source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333', ordinal: 0 }, + ]); + + expect( + sqliteJson<{ + authority_id: string; + joint_contract_participations: number; + joint_contract_value_eur: number; + }>( + dbPath, + `SELECT authority_id, joint_contract_participations, joint_contract_value_eur + FROM authority_joint_participation + WHERE authority_id IN ('auth:111111111', 'auth:222222222') + ORDER BY authority_id`, + ), + ).toEqual([ + { + authority_id: 'auth:111111111', + joint_contract_participations: 1, + joint_contract_value_eur: 100, + }, + { + authority_id: 'auth:222222222', + joint_contract_participations: 1, + joint_contract_value_eur: 100, + }, + ]); + + expect( + sqliteJson<{ spent_eur: number }>( + dbPath, + `SELECT COALESCE((SELECT spent_eur FROM authority_totals + WHERE authority_id = 'auth:111111111'), 0) AS spent_eur`, + )[0]?.spent_eur, + ).toBe(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, + ); +}); + describe('refresh-slice EOP base derivation', () => { it('derives new eop base rows as c:e contracts and is idempotent', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index ff5a3bc0c..6754cb68a 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -31,13 +31,19 @@ -- every contract has a parent. bids stays empty (the data has a bid COUNT, not bids). -- Full clear in child→parent order (D1 enforces FKs). +DROP TABLE IF EXISTS joint_tender_leads; +DROP TABLE IF EXISTS unp_prefix_authorities; +DROP TABLE IF EXISTS joint_authority_members; +DROP TABLE IF EXISTS joint_tender_sources; DELETE FROM search_index; DELETE FROM flow_pairs; DELETE FROM company_totals; +DELETE FROM authority_joint_participation; DELETE FROM authority_totals; DELETE FROM sector_totals; DELETE FROM facet_counts; DELETE FROM home_totals; +DELETE FROM contract_co_authorities; DELETE FROM contracts; DELETE FROM lots; DELETE FROM tenders; @@ -126,6 +132,186 @@ FROM ( LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; +-- 1a) Joint procurements — split the parallel EIK/name lists once and reuse the mapping for tender +-- ownership and the contract bridge. The modal raw row per UNP wins when cumulative source buckets +-- repeat a tender; ties are deterministic. Existing standalone authority names stay untouched. +CREATE TABLE joint_tender_sources ( + unp TEXT PRIMARY KEY, + authority_eiks TEXT NOT NULL, + authority_name TEXT, + authority_type TEXT +); +WITH observations AS ( + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_tenders + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + UNION ALL + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_contracts + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' +), ranked AS ( + SELECT unp, authority_eik, authority_name, authority_type, + ROW_NUMBER() OVER ( + PARTITION BY unp + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(COALESCE(authority_name, '')) DESC, + authority_eik, COALESCE(authority_name, ''), COALESCE(authority_type, '') + ) AS rn + FROM observations + GROUP BY unp, authority_eik, authority_name, authority_type +) +INSERT INTO joint_tender_sources (unp, authority_eiks, authority_name, authority_type) +SELECT unp, authority_eik, authority_name, authority_type +FROM ranked +WHERE rn = 1; + +CREATE TABLE joint_authority_members ( + unp TEXT NOT NULL, + authority_id TEXT NOT NULL, + member_name TEXT, + authority_type TEXT, + source_ordinal INTEGER NOT NULL, + PRIMARY KEY (unp, authority_id) +); +WITH RECURSIVE split ( + unp, authority_eik, member_name, authority_type, source_ordinal, eik_rest, name_rest +) AS ( + SELECT + unp, + TRIM(CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, 1, INSTR(authority_eiks, ';') - 1) ELSE authority_eiks END), + TRIM(CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, 1, INSTR(authority_name, ';') - 1) ELSE authority_name END), + authority_type, + 0, + CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, INSTR(authority_eiks, ';') + 1) ELSE '' END, + CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, INSTR(authority_name, ';') + 1) ELSE '' END + FROM joint_tender_sources + UNION ALL + SELECT + unp, + TRIM(CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, 1, INSTR(eik_rest, ';') - 1) ELSE eik_rest END), + TRIM(CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, 1, INSTR(name_rest, ';') - 1) ELSE name_rest END), + authority_type, + source_ordinal + 1, + CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, INSTR(eik_rest, ';') + 1) ELSE '' END, + CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, INSTR(name_rest, ';') + 1) ELSE '' END + FROM split + WHERE TRIM(eik_rest) <> '' +) +INSERT OR IGNORE INTO joint_authority_members + (unp, authority_id, member_name, authority_type, source_ordinal) +SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal +FROM split +-- A member EIK still carrying ';' is a composite the split could not decompose - not a real +-- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. +WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; + +-- A minority of co-authorities never occur standalone. Mint those real EIK identities from the +-- positionally corresponding name component; INSERT OR IGNORE preserves every standalone row. +WITH name_counts AS ( + SELECT authority_id, member_name, COUNT(*) AS uses, + ROW_NUMBER() OVER ( + PARTITION BY authority_id + ORDER BY COUNT(*) DESC, + CASE WHEN member_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(member_name) DESC, member_name + ) AS rn + FROM joint_authority_members + WHERE member_name IS NOT NULL + GROUP BY authority_id, member_name +), member_defaults AS ( + SELECT m.authority_id, + COALESCE(n.member_name, SUBSTR(m.authority_id, 6)) AS authority_name, + MIN(SUBSTR(m.authority_id, 6)) AS authority_eik, + MAX(m.authority_type) AS authority_type + FROM joint_authority_members m + LEFT JOIN name_counts n ON n.authority_id = m.authority_id AND n.rn = 1 + GROUP BY m.authority_id +) +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT authority_id, authority_name, authority_eik, authority_type +FROM member_defaults; + +-- Learn the modal standalone authority for each valid УНП prefix. The prefix identifies the +-- authority that registered the procedure and therefore outranks the name/first-EIK fallbacks. +CREATE TABLE unp_prefix_authorities ( + prefix TEXT PRIMARY KEY, + authority_eik TEXT NOT NULL +); +WITH prefix_observations AS ( + SELECT SUBSTR(unp, 1, 5) AS prefix, authority_eik FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + AND unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' + UNION ALL + SELECT SUBSTR(unp, 1, 5) AS prefix, authority_eik FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + AND unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' +), ranked AS ( + SELECT prefix, authority_eik, + ROW_NUMBER() OVER ( + PARTITION BY prefix + ORDER BY COUNT(*) DESC, authority_eik + ) AS rn + FROM prefix_observations + GROUP BY prefix, authority_eik +) +INSERT INTO unp_prefix_authorities (prefix, authority_eik) +SELECT prefix, authority_eik FROM ranked WHERE rn = 1; + +-- Lead default: prefer the УНП-prefix authority, then match the tender's modal raw authority name +-- to a co-authority's modal standalone name. Unmatched rows fall back to the first EIK. This +-- scratch-only mode does not alter displayed canonical names (that remains the separate #194 concern). +CREATE TABLE joint_tender_leads ( + unp TEXT PRIMARY KEY, + authority_id TEXT NOT NULL REFERENCES authorities(id) +); +WITH standalone_observations AS ( + SELECT authority_eik, authority_name FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL +), canonical_names AS ( + SELECT authority_eik, authority_name + FROM ( + SELECT authority_eik, authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, authority_name + ) AS rn + FROM standalone_observations + GROUP BY authority_eik, authority_name + ) + WHERE rn = 1 +), ranked AS ( + SELECT m.unp, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY m.unp + ORDER BY CASE WHEN p.authority_eik = SUBSTR(m.authority_id, 6) THEN 0 ELSE 1 END, + CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) AS rn + FROM joint_authority_members m + JOIN joint_tender_sources s ON s.unp = m.unp + LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) + LEFT JOIN unp_prefix_authorities p + ON p.prefix = CASE + WHEN m.unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' THEN SUBSTR(m.unp, 1, 5) + END +) +INSERT INTO joint_tender_leads (unp, authority_id) +SELECT unp, authority_id FROM ranked WHERE rn = 1; + -- 1b) Friendly authority type buckets — heuristic from name + ЗОП type (non-critical display field; -- name patterns cover Title- and UPPER-case Cyrillic since SQLite LIKE is case-sensitive for it). UPDATE authorities SET type_group = CASE @@ -162,7 +348,8 @@ SELECT 't:' || t.unp, t.unp, COALESCE(t.procurement_subject, '(без предмет)'), - 'auth:' || t.authority_eik, + COALESCE((SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik), NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, @@ -192,7 +379,13 @@ SELECT NULLIF(t.tender_id, '') -- raw EOP numeric tenderId from the header row FROM raw_tenders t WHERE t.lot_id IS NULL - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || t.authority_eik); + AND EXISTS ( + SELECT 1 FROM authorities a + WHERE a.id = COALESCE( + (SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik + ) + ); -- 2b) Synthetic tenders — УНП that appear only in contracts (no tenders-export row), so -- every contract has a parent. Procedure type is unknown ('неизвестна'); subject/CPV/ @@ -201,7 +394,10 @@ WITH folded AS ( SELECT c.unp, MIN(c.procurement_subject) AS raw_title, - 'auth:' || MIN(c.authority_eik) AS authority_id, + COALESCE( + (SELECT j.authority_id FROM joint_tender_leads j WHERE j.unp = c.unp), + 'auth:' || MIN(c.authority_eik) + ) AS authority_id, MIN(c.cpv_code) AS cpv_code, MIN(c.estimated_value) AS estimated_value, MIN(c.currency) AS raw_currency, @@ -213,7 +409,6 @@ WITH folded AS ( WHERE 1 = 1 AND c.unp IS NOT NULL AND NOT EXISTS (SELECT 1 FROM raw_tenders t WHERE t.unp = c.unp) - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || c.authority_eik) GROUP BY c.unp ) INSERT INTO tenders @@ -234,7 +429,7 @@ SELECT award_criteria, eop_tender_id FROM folded -WHERE true +WHERE EXISTS (SELECT 1 FROM authorities a WHERE a.id = folded.authority_id) ON CONFLICT(id) DO UPDATE SET title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN CASE @@ -748,6 +943,22 @@ WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); +-- Every joint contract is linked to every real co-authority. Lead is forced to ordinal 0; the +-- remaining members retain their relative source-list order. This bridge never drives money sums. +INSERT OR IGNORE INTO contract_co_authorities (contract_id, authority_id, ordinal) +SELECT contract_id, authority_id, bridge_ordinal +FROM ( + SELECT c.id AS contract_id, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY c.id + ORDER BY CASE WHEN m.authority_id = l.authority_id THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) - 1 AS bridge_ordinal + FROM contracts c + JOIN joint_authority_members m ON c.tender_id = 't:' || m.unp + JOIN joint_tender_leads l ON l.unp = m.unp +); + -- Reconciliation guard: the final summary reports the contracts inserted alongside the surviving -- staging candidates, so a future NOT NULL/foreign-key mismatch is visible instead of hidden by -- INSERT OR IGNORE. @@ -1002,6 +1213,10 @@ SELECT 1, datetime('now'); DROP TABLE contractor_identity; +DROP TABLE joint_tender_leads; +DROP TABLE unp_prefix_authorities; +DROP TABLE joint_authority_members; +DROP TABLE joint_tender_sources; -- Summary (last result set printed by `wrangler d1 execute`) SELECT @@ -1012,6 +1227,7 @@ SELECT (SELECT COUNT(*) FROM bidders WHERE eik_valid = 0) AS bidders_name_keyed, (SELECT COUNT(*) FROM bidders WHERE kind = 'consortium') AS consortia, (SELECT COUNT(*) FROM contracts) AS contracts, + (SELECT COUNT(*) FROM contract_co_authorities) AS joint_participations, (SELECT contract_candidates FROM pipeline_stats) AS contract_candidates, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'value_suspect') AS value_suspect, (SELECT COUNT(*) FROM contracts WHERE value_flag = 'annex_suspect') AS annex_suspect, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index 34357a01a..bdecce18a 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -77,6 +77,8 @@ CREATE TABLE IF NOT EXISTS authority_totals ( eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT ); DELETE FROM authority_totals; +-- Spend and the ordinary contract count remain lead-only, preserving the reconciliation invariant: +-- SUM(authority_totals.spent_eur) = the tender-authority-attributed contract sum. INSERT INTO authority_totals (authority_id, name, type_group, settlement, region, spent_eur, contracts, suppliers, avg_eur, eu_eur, first_date, last_date) SELECT a.id, a.name, a.type_group, a.settlement, a.region, SUM(c.amount_eur), COUNT(*), COUNT(DISTINCT c.bidder_id), SUM(c.amount_eur) / COUNT(*), @@ -90,6 +92,21 @@ UPDATE authority_totals SET primary_sector = ( WHERE t.authority_id = authority_totals.authority_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1); +-- Separate participation metrics. They count every joint-contract bridge association, including +-- the lead; the value is informational only and never feeds authority_totals or any national total. +CREATE TABLE IF NOT EXISTS authority_joint_participation ( + authority_id TEXT PRIMARY KEY REFERENCES authorities(id), + joint_contract_participations INTEGER NOT NULL, + joint_contract_value_eur REAL NOT NULL DEFAULT 0 +); +DELETE FROM authority_joint_participation; +INSERT INTO authority_joint_participation + (authority_id, joint_contract_participations, joint_contract_value_eur) +SELECT cca.authority_id, COUNT(*), COALESCE(SUM(c.amount_eur), 0) +FROM contract_co_authorities cca +JOIN contracts c ON c.id = cca.contract_id +GROUP BY cca.authority_id; + -- home_totals uses the browsable leaderboard grains for authority/bidder counts, and the same -- freshness definition as refresh-slice.sql: latest in-corpus signed contract date. INSERT INTO home_totals (id, contracts, value_eur, authorities, bidders, suspect, first_date, last_date, as_of, refreshed_at) diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 725efdc44..8b7a5f671 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -16,6 +16,10 @@ CREATE INDEX IF NOT EXISTS idx_contracts_tender_id ON contracts(tender_id); DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; +DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_unp_prefix_authorities; +DROP TABLE IF EXISTS refresh_joint_authority_members; +DROP TABLE IF EXISTS refresh_joint_tender_sources; CREATE TABLE refresh_touched_contracts (id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_bidders (bidder_id TEXT PRIMARY KEY); CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); @@ -46,7 +50,8 @@ FROM ( SELECT source, authority_eik, authority_name FROM raw_tenders ) WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') - AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + AND authority_eik IS NOT NULL + AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL AND TRIM(authority_name) <> '' GROUP BY authority_eik, authority_name ) @@ -93,16 +98,189 @@ SELECT s.authority_eik, act.canonical_type FROM ( - -- Composite joint EIKs must not mint standalone authorities (see normalize-raw.sql). SELECT authority_eik FROM raw_contracts - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + AND authority_eik NOT LIKE '%;%' UNION SELECT authority_eik FROM raw_tenders - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + AND authority_eik NOT LIKE '%;%' ) s LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; +-- Split joint EIK/name lists once for lead selection and bridge population. This is the scoped +-- mirror of normalize-raw.sql step 1a; it deliberately does not rewrite existing display names. +CREATE TABLE refresh_joint_tender_sources ( + unp TEXT PRIMARY KEY, + authority_eiks TEXT NOT NULL, + authority_name TEXT, + authority_type TEXT +); +WITH observations AS ( + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_tenders + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') + UNION ALL + SELECT unp, authority_eik, authority_name, authority_type + FROM raw_contracts + WHERE unp IS NOT NULL AND authority_eik LIKE '%;%' + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') +), ranked AS ( + SELECT unp, authority_eik, authority_name, authority_type, + ROW_NUMBER() OVER ( + PARTITION BY unp + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(COALESCE(authority_name, '')) DESC, + authority_eik, COALESCE(authority_name, ''), COALESCE(authority_type, '') + ) AS rn + FROM observations + GROUP BY unp, authority_eik, authority_name, authority_type +) +INSERT INTO refresh_joint_tender_sources (unp, authority_eiks, authority_name, authority_type) +SELECT unp, authority_eik, authority_name, authority_type FROM ranked WHERE rn = 1; + +CREATE TABLE refresh_joint_authority_members ( + unp TEXT NOT NULL, + authority_id TEXT NOT NULL, + member_name TEXT, + authority_type TEXT, + source_ordinal INTEGER NOT NULL, + PRIMARY KEY (unp, authority_id) +); +WITH RECURSIVE split ( + unp, authority_eik, member_name, authority_type, source_ordinal, eik_rest, name_rest +) AS ( + SELECT + unp, + TRIM(CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, 1, INSTR(authority_eiks, ';') - 1) ELSE authority_eiks END), + TRIM(CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, 1, INSTR(authority_name, ';') - 1) ELSE authority_name END), + authority_type, + 0, + CASE WHEN INSTR(authority_eiks, ';') > 0 + THEN SUBSTR(authority_eiks, INSTR(authority_eiks, ';') + 1) ELSE '' END, + CASE WHEN INSTR(COALESCE(authority_name, ''), ';') > 0 + THEN SUBSTR(authority_name, INSTR(authority_name, ';') + 1) ELSE '' END + FROM refresh_joint_tender_sources + UNION ALL + SELECT + unp, + TRIM(CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, 1, INSTR(eik_rest, ';') - 1) ELSE eik_rest END), + TRIM(CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, 1, INSTR(name_rest, ';') - 1) ELSE name_rest END), + authority_type, + source_ordinal + 1, + CASE WHEN INSTR(eik_rest, ';') > 0 + THEN SUBSTR(eik_rest, INSTR(eik_rest, ';') + 1) ELSE '' END, + CASE WHEN INSTR(name_rest, ';') > 0 + THEN SUBSTR(name_rest, INSTR(name_rest, ';') + 1) ELSE '' END + FROM split + WHERE TRIM(eik_rest) <> '' +) +INSERT OR IGNORE INTO refresh_joint_authority_members + (unp, authority_id, member_name, authority_type, source_ordinal) +SELECT unp, 'auth:' || authority_eik, NULLIF(member_name, ''), authority_type, source_ordinal +FROM split +-- A member EIK still carrying ';' is a composite the split could not decompose - not a real +-- single authority; it must not seed a member row or mint an orphan 'auth:EIK1; EIK2'. +WHERE authority_eik <> '' AND authority_eik NOT LIKE '%;%'; + +WITH name_counts AS ( + SELECT authority_id, member_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_id + ORDER BY COUNT(*) DESC, + CASE WHEN member_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(member_name) DESC, member_name + ) AS rn + FROM refresh_joint_authority_members + WHERE member_name IS NOT NULL + GROUP BY authority_id, member_name +), member_defaults AS ( + SELECT m.authority_id, + COALESCE(n.member_name, SUBSTR(m.authority_id, 6)) AS authority_name, + MIN(SUBSTR(m.authority_id, 6)) AS authority_eik, + MAX(m.authority_type) AS authority_type + FROM refresh_joint_authority_members m + LEFT JOIN name_counts n ON n.authority_id = m.authority_id AND n.rn = 1 + GROUP BY m.authority_id +) +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT authority_id, authority_name, authority_eik, authority_type FROM member_defaults; + +-- Learn the modal authority for each valid УНП prefix from the full existing tender history. Raw +-- staging contains only the touched slice here, so it cannot provide a stable corpus-wide map. +CREATE TABLE refresh_unp_prefix_authorities ( + prefix TEXT PRIMARY KEY, + authority_eik TEXT NOT NULL +); +WITH prefix_observations AS ( + SELECT SUBSTR(source_id, 1, 5) AS prefix, SUBSTR(authority_id, 6) AS authority_eik + FROM tenders + WHERE authority_id NOT LIKE '%;%' + AND source_id GLOB '[0-9][0-9][0-9][0-9][0-9]-*' +), ranked AS ( + SELECT prefix, authority_eik, + ROW_NUMBER() OVER ( + PARTITION BY prefix + ORDER BY COUNT(*) DESC, authority_eik + ) AS rn + FROM prefix_observations + GROUP BY prefix, authority_eik +) +INSERT INTO refresh_unp_prefix_authorities (prefix, authority_eik) +SELECT prefix, authority_eik FROM ranked WHERE rn = 1; + +CREATE TABLE refresh_joint_tender_leads ( + unp TEXT PRIMARY KEY, + authority_id TEXT NOT NULL REFERENCES authorities(id) +); +WITH standalone_observations AS ( + SELECT authority_eik, authority_name FROM raw_contracts + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders + WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL + AND (source LIKE 'eop:%' OR source LIKE 'ocds:%') +), canonical_names AS ( + SELECT authority_eik, authority_name + FROM ( + SELECT authority_eik, authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY COUNT(*) DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, authority_name + ) AS rn + FROM standalone_observations + GROUP BY authority_eik, authority_name + ) + WHERE rn = 1 +), ranked AS ( + SELECT m.unp, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY m.unp + ORDER BY CASE WHEN p.authority_eik = SUBSTR(m.authority_id, 6) THEN 0 ELSE 1 END, + CASE WHEN c.authority_name = s.authority_name THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) AS rn + FROM refresh_joint_authority_members m + JOIN refresh_joint_tender_sources s ON s.unp = m.unp + LEFT JOIN canonical_names c ON c.authority_eik = SUBSTR(m.authority_id, 6) + LEFT JOIN refresh_unp_prefix_authorities p + ON p.prefix = CASE + WHEN m.unp GLOB '[0-9][0-9][0-9][0-9][0-9]-*' THEN SUBSTR(m.unp, 1, 5) + END +) +INSERT INTO refresh_joint_tender_leads (unp, authority_id) +SELECT unp, authority_id FROM ranked WHERE rn = 1; + -- type_group for any authority still missing it (covers the rows just inserted) — same heuristic as -- normalize-raw.sql step 1b. UPDATE authorities SET type_group = CASE @@ -342,7 +520,8 @@ SELECT 't:' || t.unp, t.unp, COALESCE(t.procurement_subject, '(без предмет)'), - 'auth:' || t.authority_eik, + COALESCE((SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik), NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, @@ -372,7 +551,13 @@ SELECT NULLIF(t.tender_id, '') -- raw EOP numeric tenderId from the header row FROM raw_tenders t WHERE t.lot_id IS NULL - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || t.authority_eik) + AND EXISTS ( + SELECT 1 FROM authorities a + WHERE a.id = COALESCE( + (SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = t.unp), + 'auth:' || t.authority_eik + ) + ) ON CONFLICT(id) DO UPDATE SET source_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.source_id ELSE tenders.source_id END, title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.title ELSE tenders.title END, @@ -550,7 +735,10 @@ WITH folded AS ( SELECT c.unp, MIN(c.procurement_subject) AS raw_title, - 'auth:' || MIN(c.authority_eik) AS authority_id, + COALESCE( + (SELECT j.authority_id FROM refresh_joint_tender_leads j WHERE j.unp = c.unp), + 'auth:' || MIN(c.authority_eik) + ) AS authority_id, MIN(c.cpv_code) AS cpv_code, MIN(c.estimated_value) AS estimated_value, MIN(c.currency) AS raw_currency, @@ -562,7 +750,6 @@ WITH folded AS ( WHERE (c.source LIKE 'eop:%' OR c.source LIKE 'ocds:%') AND c.unp IS NOT NULL AND NOT EXISTS (SELECT 1 FROM raw_tenders t WHERE t.unp = c.unp) - AND EXISTS (SELECT 1 FROM authorities a WHERE a.id = 'auth:' || c.authority_eik) GROUP BY c.unp ) INSERT INTO tenders @@ -583,7 +770,7 @@ SELECT award_criteria, eop_tender_id FROM folded -WHERE true +WHERE EXISTS (SELECT 1 FROM authorities a WHERE a.id = folded.authority_id) ON CONFLICT(id) DO UPDATE SET title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN CASE @@ -688,6 +875,10 @@ SELECT DISTINCT t.authority_id FROM contracts c JOIN tenders t ON t.id = c.tender_id WHERE c.id IN (SELECT id FROM refresh_touched_contracts) AND t.authority_id IS NOT NULL; +INSERT OR IGNORE INTO refresh_touched_authorities (authority_id) +SELECT DISTINCT ca.authority_id +FROM contract_co_authorities ca +WHERE ca.contract_id IN (SELECT id FROM refresh_touched_contracts); DELETE FROM contracts WHERE id IN ( @@ -1188,6 +1379,24 @@ WHERE x.display_native IS NOT NULL AND c2.bidder_id = x.bidder_key ); +-- Rebuild joint-participation links for contracts represented by this window. Contract deletes +-- cascade old bridge rows; this inserts every member again with the lead forced to ordinal 0. +INSERT OR IGNORE INTO contract_co_authorities (contract_id, authority_id, ordinal) +SELECT contract_id, authority_id, bridge_ordinal +FROM ( + SELECT c.id AS contract_id, m.authority_id, + ROW_NUMBER() OVER ( + PARTITION BY c.id + ORDER BY CASE WHEN m.authority_id = l.authority_id THEN 0 ELSE 1 END, + m.source_ordinal, m.authority_id + ) - 1 AS bridge_ordinal + FROM contracts c + JOIN refresh_joint_authority_members m ON c.tender_id = 't:' || m.unp + JOIN refresh_joint_tender_leads l ON l.unp = m.unp +); +INSERT OR IGNORE INTO refresh_touched_authorities (authority_id) +SELECT DISTINCT authority_id FROM refresh_joint_authority_members; + UPDATE tenders SET status = 'awarded' WHERE status <> 'awarded' @@ -1468,6 +1677,17 @@ UPDATE authority_totals SET primary_sector = ( GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1) WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); +-- Joint participation is separate from lead totals; its value is informational and non-summable. +DELETE FROM authority_joint_participation +WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); +INSERT INTO authority_joint_participation + (authority_id, joint_contract_participations, joint_contract_value_eur) +SELECT cca.authority_id, COUNT(*), COALESCE(SUM(c.amount_eur), 0) +FROM contract_co_authorities cca +JOIN contracts c ON c.id = cca.contract_id +WHERE cca.authority_id IN (SELECT authority_id FROM refresh_touched_authorities) +GROUP BY cca.authority_id; + -- @refresh-batch flow-pairs DELETE FROM flow_pairs; INSERT INTO flow_pairs (authority_id, bidder_id, authority_name, bidder_name, bidder_kind, won_eur, contracts) @@ -1542,6 +1762,10 @@ SELECT 'eu', CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END, COUNT(*), COALESCE FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; -- @refresh-batch cleanup +DROP TABLE IF EXISTS refresh_joint_tender_leads; +DROP TABLE IF EXISTS refresh_unp_prefix_authorities; +DROP TABLE IF EXISTS refresh_joint_authority_members; +DROP TABLE IF EXISTS refresh_joint_tender_sources; DROP TABLE IF EXISTS refresh_touched_contracts; DROP TABLE IF EXISTS refresh_touched_bidders; DROP TABLE IF EXISTS refresh_touched_authorities; diff --git a/scripts/ship-domain.mjs b/scripts/ship-domain.mjs index 2e3b83c96..96aa9200d 100755 --- a/scripts/ship-domain.mjs +++ b/scripts/ship-domain.mjs @@ -16,6 +16,7 @@ const TABLES = [ 'tenders', 'lots', 'contracts', + 'contract_co_authorities', 'amendments', 'parties', 'fx_rates',