From fc1878f4d26b50176fcd21e16e0aa110575bc2bc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 02:42:32 +0000 Subject: [PATCH 1/9] fix(etl): enforce bulstat checksum for eik identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit eik_valid now computes the real Bulstat control digit (9-digit weights 1..8 mod 11 with the remainder-10 reweight, 13-digit second pass), and rejects the degenerate all-zero code. A checksum-invalid ЕИК routes the bidder to a name: key instead of eik:, so falsely-merged distinct companies split apart; a valid ЕИК stays merged correctly. The checksum block is repeated across both ETL paths and pinned identical by a consistency test; fixtures updated to checksum-valid ЕИК-та. Extracts the checksum fix from #203 (@StanislavBG). --- packages/db/src/competition-sql.test.ts | 6 +- .../db/src/eik-checksum-consistency.test.ts | 33 ++++ packages/db/src/queries/companies.test.ts | 10 +- .../src/queries/contracts-filter-sql.test.ts | 10 +- packages/db/src/queries/contracts.test.ts | 4 +- packages/db/src/queries/details.test.ts | 10 +- packages/db/src/queries/identity.test.ts | 2 +- packages/db/src/refresh-slice.test.ts | 128 ++++++++------- packages/db/src/ship-domain.test.ts | 4 +- scripts/normalize-raw.sql | 126 +++++++++++--- scripts/refresh-slice.sql | 154 +++++++++++++++--- 11 files changed, 359 insertions(+), 128 deletions(-) create mode 100644 packages/db/src/eik-checksum-consistency.test.ts diff --git a/packages/db/src/competition-sql.test.ts b/packages/db/src/competition-sql.test.ts index baa0fdb48..20d05aad2 100644 --- a/packages/db/src/competition-sql.test.ts +++ b/packages/db/src/competition-sql.test.ts @@ -31,10 +31,10 @@ function readScript(dbPath: string, path: string): void { const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:A', 'Институция А', '100000001', 'община'), - ('auth:B', 'Институция Б', '100000002', 'община'); + ('auth:B', 'Институция Б', '100000019', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:X', 'Фирма Х', '200000001', '200000001', 1, 'company'), - ('eik:Y', 'Фирма У', '200000002', '200000002', 1, 'company'); + ('eik:X', 'Фирма Х', '200000002', '200000002', 1, 'company'), + ('eik:Y', 'Фирма У', '200000015', '200000015', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:A', '45', 'открита процедура', 'awarded'), ('t:B', 'UNP-B', 'Поръчка Б', 'auth:B', '45', 'открита процедура', 'awarded'); diff --git a/packages/db/src/eik-checksum-consistency.test.ts b/packages/db/src/eik-checksum-consistency.test.ts new file mode 100644 index 000000000..606cf4e03 --- /dev/null +++ b/packages/db/src/eik-checksum-consistency.test.ts @@ -0,0 +1,33 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); + +function checksumBlocks(path: string): string[] { + const sql = readFileSync(path, 'utf8'); + const pattern = + /^[ \t]*CASE\r?\n(?=[ \t]+WHEN eik_clean IS NULL\b[^\r\n]*LENGTH\(eik_clean\) NOT IN \(9, 13\) THEN 0$)[\s\S]*?^[ \t]*END AS eik_valid$/gm; + + return [...sql.matchAll(pattern)].map(([block]) => { + const lines = block.split(/\r?\n/); + const indentation = Math.min( + ...lines.filter((line) => line.trim()).map((line) => line.match(/^[ \t]*/)?.[0].length ?? 0), + ); + return lines.map((line) => line.slice(indentation)).join('\n'); + }); +} + +describe('ЕИК checksum copies stay consistent', () => { + it('keeps every checksum CASE block textually identical', () => { + const normalizeBlocks = checksumBlocks(resolve(root, 'scripts/normalize-raw.sql')); + const refreshBlocks = checksumBlocks(resolve(root, 'scripts/refresh-slice.sql')); + const blocks = [...normalizeBlocks, ...refreshBlocks]; + + expect(normalizeBlocks).toHaveLength(3); + expect(refreshBlocks).toHaveLength(4); + expect(new Set(blocks)).toEqual(new Set([blocks[0]])); + }); +}); diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index a34c1ad07..a4643d026 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -4,11 +4,11 @@ import type { CompanyTotalsRow } from './rows'; const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ { - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', name: 'Филтрирана фирма', kind: 'company', ownership_kind: null, - eik: '111111111', + eik: '111111113', eik_valid: 1, settlement: 'София', won_eur: 1000, @@ -25,11 +25,11 @@ const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ const unfilteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ ...filteredRows, { - bidder_id: 'eik:999999999', + bidder_id: 'eik:999999995', name: 'Нефилтрирана фирма', kind: 'company', ownership_kind: null, - eik: '999999999', + eik: '999999995', eik_valid: 1, settlement: 'Пловдив', won_eur: 900, @@ -93,7 +93,7 @@ describe('streamCompaniesCsv', () => { .map((line) => line.split(',')[0]); expect(csvEiks).toEqual(list.items.map((item) => item.eik)); - expect(csvEiks).toEqual(['111111111']); + expect(csvEiks).toEqual(['111111113']); }); it('exports fewer rows for a sector filter than for the unfiltered corpus', async () => { diff --git a/packages/db/src/queries/contracts-filter-sql.test.ts b/packages/db/src/queries/contracts-filter-sql.test.ts index 2c4d69746..98314206e 100644 --- a/packages/db/src/queries/contracts-filter-sql.test.ts +++ b/packages/db/src/queries/contracts-filter-sql.test.ts @@ -26,14 +26,14 @@ const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:100000001', 'Институция А', '100000001', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:200000001', 'Фирма Х', '200000001', '200000001', 1, 'company'); + ('eik:200000002', 'Фирма Х', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:100000001', '45000000', 'открита процедура', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, bids_received, value_flag, amount_eur) VALUES - ('c:NULL', 't:A', 'eik:200000001', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), - ('c:ZERO', 't:A', 'eik:200000001', 200, 'EUR', '2024-01-02', 0, 'ok', 200), - ('c:ONE', 't:A', 'eik:200000001', 300, 'EUR', '2024-01-03', 1, 'ok', 300), - ('c:TWO', 't:A', 'eik:200000001', 400, 'EUR', '2024-01-04', 2, 'ok', 400); + ('c:NULL', 't:A', 'eik:200000002', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), + ('c:ZERO', 't:A', 'eik:200000002', 200, 'EUR', '2024-01-02', 0, 'ok', 200), + ('c:ONE', 't:A', 'eik:200000002', 300, 'EUR', '2024-01-03', 1, 'ok', 300), + ('c:TWO', 't:A', 'eik:200000002', 400, 'EUR', '2024-01-04', 2, 'ok', 400); `; /** Minimal D1Database facade over node:sqlite — enough for the query layer's prepare/bind/all/first. */ diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 504754b24..d23391116 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -21,9 +21,9 @@ const contractRow = { unp: 'UNP-1', cpv_code: '45000000', eu_funded: 0, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, procedure_type: 'Открита процедура', diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 04e3cd009..ffe35cffe 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -41,14 +41,14 @@ const baseContractRow = { tender_fx_rate: null as number | null, start_date: null, end_date: null, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', authority_type_group: 'ministry', authority_settlement: 'Sofia', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, - bidder_eik: '111111111', + bidder_eik: '111111113', bidder_settlement: 'Sofia', }; @@ -102,7 +102,7 @@ describe('getContract', () => { estimated_fx_rate: null, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', @@ -147,7 +147,7 @@ describe('getContract', () => { estimated_fx_rate: 0.9, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', diff --git a/packages/db/src/queries/identity.test.ts b/packages/db/src/queries/identity.test.ts index 0cdc42726..2acc6ccfb 100644 --- a/packages/db/src/queries/identity.test.ts +++ b/packages/db/src/queries/identity.test.ts @@ -23,7 +23,7 @@ describe('company slug', () => { expect(bidderIdFromSlug(slug)).toBe(id); }); it('round-trips a 13-digit ЕИК', () => { - expect(bidderIdFromSlug(companySlug('eik:1234567890123'))).toBe('eik:1234567890123'); + expect(bidderIdFromSlug(companySlug('eik:8316417910124'))).toBe('eik:8316417910124'); }); }); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76f..3318424cf 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -64,11 +64,11 @@ INSERT INTO raw_tenders VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -82,12 +82,12 @@ INSERT INTO raw_contracts VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-CE-1', '2026-06-01', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', - 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', '1', 'CONTRACT-CE-1', '2026-06-02', 1000, 'BGN', - 'Base contract', 0, '987654321', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Base contract', 0, '987654308', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', 0, 'DOC-CO-1', '2026-06-01', 'OCDS-CO-1', 'TENDER-CO-1', 'open', 'OCDS tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', NULL, 'CONTRACT-CO-1', '2026-06-02', 1000, 'BGN', 'OCDS contract', 0, '987654322', 'Bidder CO', 'BG', 'small', 0, 3, 1, 0, 0, 30); @@ -98,10 +98,10 @@ INSERT INTO raw_amendments currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase'), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', '1', 'AMD-CO-1', - 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456789', 'Authority CE', + 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456786', 'Authority CE', 'OCDS tender', 'works', 1000, 1300, 300, 'BGN', 'OCDS increase'); `, ); @@ -117,7 +117,7 @@ function seedRepeatedAnnexOnly(dbPath: string): void { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase');`, ); } @@ -133,7 +133,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', 'Construction', 'works', 5000, 'BGN', 'basis', - 'lowest', 'Authority Shared', '223456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Shared', '223456787', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -147,7 +147,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-SHARED', '2026-06-01', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -167,7 +167,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { VALUES ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-SHARED-O', '2026-06-02', 'OCDS-SHARED', 'TENDER-SHARED-O', 'open', 'Shared tender ocds', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract ocds', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -189,11 +189,11 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -207,29 +207,29 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-OLD', '2026-06-01', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'Old duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Old duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NEW', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'New duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'New duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NULL', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, NULL, '2026-06-03', 500, 'BGN', 'Null number', 0, '577777778', 'Bidder Null', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS', '2026-06-02', 'OCDS-ID', 'TENDER-OCDS', 'open', 'OCDS ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', 'LOT-0001', 'CONTRACT-OCDS', '2026-06-04', 700, 'BGN', - 'OCDS ID', 0, '577777779', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), + 'OCDS ID', 0, '577777792', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS-SKIP', '2026-06-02', 'OCDS-ID-SKIP', 'TENDER-OCDS-SKIP', 'open', 'OCDS skip tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-ID', '2026-06-04', 700, 'BGN', - 'OCDS skip', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, + 'OCDS skip', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, ); } @@ -259,7 +259,7 @@ function seedSyntheticWindow( '2026-06-02', ${sqlValue(values.unp)}, ${sqlValue(`TENDER-${values.unp}`)}, 'open', ${sqlValue(values.subject)}, ${sqlValue(values.cpv)}, 'Construction', 'works', ${sqlValue(values.estimated)}, ${sqlValue(values.currency)}, - 'basis', 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', 'notice', + 'basis', 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${values.source}`)}, '2026-06-02', 100, ${sqlValue(values.currency)}, 'Synthetic contract', 0, '667777777', 'Bidder Synthetic', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -269,7 +269,7 @@ function seedSyntheticAuthority(dbPath: string): void { sqlite( dbPath, `INSERT OR IGNORE INTO authorities (id, name, bulstat, type, type_group) - VALUES ('auth:623456789', 'Authority Synthetic', '623456789', 'public', 'other');`, + VALUES ('auth:623456780', 'Authority Synthetic', '623456780', 'public', 'other');`, ); } @@ -281,7 +281,7 @@ function seedReplayParty(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-replay', 'party-replay', - '823456789', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', + '823456782', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', 'replay@example.test', '+359000000001'), ('ocds:2026-06-01', '2026-06-08T00:00:00Z', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);`, @@ -302,9 +302,9 @@ function seedReplayEntity(dbPath: string): void { VALUES ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-REPLAY', '2026-06-02', 'UNP-REPLAY', 'TENDER-REPLAY', 'open', 'Replay tender', '45000000', - 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456789', + 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-REPLAY', '2026-06-02', 100, 'BGN', - 'Replay contract', 0, '823456789', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, + 'Replay contract', 0, '823456782', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); } @@ -320,16 +320,16 @@ function seedCollidingPartySlot(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '111111111', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), + '111111113', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), ('ocds:2026-06-05', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '222222222', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, + '222222226', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, ); } function seedTwoCollidingBidders(dbPath: string): void { for (const [eik, name] of [ - ['111111111', 'Company A'], - ['222222222', 'Company B'], + ['111111113', 'Company A'], + ['222222226', 'Company B'], ] as const) { sqlite( dbPath, @@ -345,7 +345,7 @@ function seedTwoCollidingBidders(dbPath: string): void { (${sqlValue(`eop:contracts:2026-06-02:${eik}`)}, 2026, 'eop', '2026-06-08T00:00:00Z', 0, ${sqlValue(`DOC-${eik}`)}, '2026-06-02', ${sqlValue(`UNP-${eik}`)}, ${sqlValue(`TENDER-${eik}`)}, 'open', 'Slot tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', - 'Slot authority', '923456789', 'public', 'activity', 'notice', NULL, + 'Slot authority', '923456783', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${eik}`)}, '2026-06-02', 100, 'BGN', 'Slot contract', 0, ${sqlValue(eik)}, ${sqlValue(name)}, 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -488,15 +488,15 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, - `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456789', 'Authority Flag', '323456789', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777777', 'Bidder Flag', '777777777', '777777777', 1, 'company'); + `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456788', 'Authority Flag', '323456788', 'public'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777771', 'Bidder Flag', '777777771', '777777771', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) - VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456789', 1000, 'BGN', 'open', 'awarded'); + VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456788', 1000, 'BGN', 'open', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, value_flag, amount_eur, signing_value_eur) VALUES - ('c:e:flag', 't:UNP-FLAG', 'eik:777777777', 100, 'BGN', '2026-06-02', + ('c:e:flag', 't:UNP-FLAG', 'eik:777777771', 100, 'BGN', '2026-06-02', 'CONTRACT-FLAG', 100, 10000, 1, 'annex_suspect', NULL, 100 / 1.95583); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -505,7 +505,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '2', 'AMD-FLAG-2', - 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456789', 'Authority Flag', + 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456788', 'Authority Flag', 'Flag tender', 'works', 10000, 120, -9880, 'BGN', 'Normalize');`, ); @@ -560,7 +560,7 @@ describe('refresh-slice EOP base derivation', () => { '2026-06-01', 'UNP-MISMATCH', 'TENDER-MISMATCH', 'open', 'Mismatch tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Authority Mismatch', '423456789', 'public', 'activity', 'notice', NULL, 'CONTRACT-MISMATCH', '2026-06-02', 1000, 'BGN', - 'Mismatch contract', 0, '677777777', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); + 'Mismatch contract', 0, '677777779', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -660,7 +660,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority Ordinal', '723456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_amendments @@ -670,7 +670,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-ORD', - 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456789', 'Authority Ordinal', + 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456781', 'Authority Ordinal', 'Ordinal tender', 'works', 100, 150, 50, 'BGN', 'Increase');`, ); } @@ -687,12 +687,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -709,12 +709,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -833,7 +833,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-REAL', 'TENDER-REAL', NULL, 'Real header', '99000000', 'Header', 'works', 990, 'EUR', 'basis', - 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01');`, ); seedSyntheticWindow(dbPath, { @@ -884,11 +884,17 @@ describe('refresh-slice EOP base derivation', () => { readScript(sliceDb, refreshSlicePath); const authoritySql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456789'"; + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456782'"; const bidderSql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456789'"; - expect(sqliteJson(fullDb, authoritySql)).toEqual(sqliteJson(sliceDb, authoritySql)); - expect(sqliteJson(fullDb, bidderSql)).toEqual(sqliteJson(sliceDb, bidderSql)); + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456782'"; + const fullAuthorities = sqliteJson(fullDb, authoritySql); + const sliceAuthorities = sqliteJson(sliceDb, authoritySql); + const fullBidders = sqliteJson(fullDb, bidderSql); + const sliceBidders = sqliteJson(sliceDb, bidderSql); + expect(fullAuthorities).not.toHaveLength(0); + expect(sliceAuthorities).toEqual(fullAuthorities); + expect(fullBidders).not.toHaveLength(0); + expect(sliceBidders).toEqual(fullBidders); const partiesAfterWindow2 = sqliteJson(sliceDb, 'SELECT * FROM parties ORDER BY party_key'); const authorityAfterWindow2 = sqliteJson(sliceDb, authoritySql); @@ -920,19 +926,19 @@ describe('refresh-slice EOP base derivation', () => { const sliceDb = resolve(dir, 'slice.sqlite'); try { const partyByEik = - "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111111','222222222') ORDER BY eik"; + "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111113','222222226') ORDER BY eik"; const bidderByEik = - "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111111','222222222') ORDER BY eik_normalized"; + "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111113','222222226') ORDER BY eik_normalized"; // Each company must be enriched from its OWN party row, not the latest slot occupant's. const expectedParties = [ { - eik: '111111111', + eik: '111111113', locality: 'City A', street_address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', locality: 'City B', street_address: 'Addr B 2', contact_email: 'b@example.test', @@ -940,13 +946,13 @@ describe('refresh-slice EOP base derivation', () => { ]; const expectedBidders = [ { - eik: '111111111', + eik: '111111113', settlement: 'City A', address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', settlement: 'City B', address: 'Addr B 2', contact_email: 'b@example.test', @@ -988,7 +994,7 @@ function seedReattrContract(dbPath: string, source: string, eik: string, name: s VALUES (${sqlValue(source)}, 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-REATTR', '2026-06-02', 'UNP-REATTR', 'TENDER-REATTR', 'open', 'Reattr tender', '45000000', - 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Reattr authority', '923456789', + 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Reattr authority', '923456783', 'public', 'activity', 'notice', NULL, 'CONTRACT-REATTR', '2026-06-02', 100, 'BGN', 'Reattr contract', 0, ${sqlValue(eik)}, ${sqlValue(name)}, 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -1012,18 +1018,18 @@ describe('refresh-slice integrity gate', () => { try { initWorkDb(dbPath); // window 1: contract attributed to bidder A - seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111111', 'Company A'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111113', 'Company A'); readScript(dbPath, refreshSlicePath); - expect(wonEur('111111111')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBeGreaterThan(0); // window 2: same contract_number+unp re-imported with a different contractor → re-attribution resetRawStaging(dbPath); - seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222222', 'Company B'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222226', 'Company B'); readScript(dbPath, refreshSlicePath); // A's scoped rollup was rebuilt to empty (the value moved), B now carries it… - expect(wonEur('111111111')).toBe(0); - expect(wonEur('222222222')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBe(0); + expect(wonEur('222222226')).toBeGreaterThan(0); // …so the gate reconciles on the slice-built DB (rollup check runs; staging self-skips here). const results = assertIntegrity(run, { label: 'test-slice', exit: false }); diff --git a/packages/db/src/ship-domain.test.ts b/packages/db/src/ship-domain.test.ts index e1249ed7d..7101ec83c 100644 --- a/packages/db/src/ship-domain.test.ts +++ b/packages/db/src/ship-domain.test.ts @@ -49,9 +49,9 @@ describe('ship-domain', () => { workDb, `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:1', 'Authority line 1 Authority line 2', '1', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000007', 'Bidder', '200000007', '200000007', 1, 'company'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000002', 'Bidder', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, currency, procedure_type, status) VALUES ('t:1', '1', 'Tender', 'auth:1', 'BGN', 'open', 'awarded'); - INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000007', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); + INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000002', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); INSERT INTO amendments (id, natural_key, contract_number, unp, description, source) VALUES ('am:1:C1:A1', 'am:1:C1:A1', 'C1', '1', 'Description line 1 Description line 2', 'test'); INSERT INTO nuts_regions (nuts3, nuts3_name, nuts2, nuts2_name, nuts1, nuts1_name) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66ed..74ce5bf4f 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -230,7 +230,7 @@ WHERE t.lot_id IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || t.unp); -- 4) Bidders — winning contractors, with a robust identity key: --- * valid ЕИК (digits-only, length 9/13) → key by ЕИК ('eik:') +-- * checksum-valid Bulgarian ЕИК → key by ЕИК ('eik:') -- * otherwise (withheld 'не се публикува', foreign id, multi-ЕИК consortium, missing) → -- key by NORMALISED NAME ('name:') -- Keying invalid ЕИК by name stops the collapse where ~595 distinct withheld-ЕИК contractors @@ -251,9 +251,9 @@ FROM ( SELECT contractor_name, eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, + eik_valid, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key, @@ -267,8 +267,42 @@ FROM ( FROM ( SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL @@ -489,13 +523,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -757,13 +818,40 @@ SELECT 1, WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed2..a145895c5 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -56,10 +56,9 @@ SELECT MAX(grp), CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END FROM ( - SELECT contractor_name, eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, + SELECT contractor_name, eik_clean, eik_valid, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key, @@ -71,9 +70,33 @@ FROM ( THEN 1 ELSE 0 END AS grp FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + SELECT contractor_name, eik_clean, + -- Bulgarian ЕИК/Булстат checksum — identical to normalize-raw.sql step 4 (#195), so the + -- incremental path's bidder identity stays in parity with the full rebuild. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL @@ -679,13 +702,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -927,13 +977,40 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + ( + SELECT + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END + FROM ( + SELECT + eik_clean, + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean + ) + ) + ) AS bidder_key FROM ( SELECT c.*, CASE @@ -1244,16 +1321,43 @@ WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NUL OR b.id IN ( SELECT bidder_key FROM ( - SELECT contractor_name, eik_clean, + SELECT + contractor_name, CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN eik_valid = 1 THEN 'eik:' || eik_clean WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) ELSE NULL END AS bidder_key FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + SELECT + contractor_name, + eik_clean, + -- Same Bulgarian ЕИК checksum as the bidder INSERT above and normalize-raw.sql (#195), + -- so the touched-bidder key matches the actual bidders.id keying. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) ) ) WHERE bidder_key IS NOT NULL From ce144331140f38a12d90a2796ff086f1e8975441 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 02:44:02 +0000 Subject: [PATCH 2/9] feat(etl): canonical entity fields by frequency mode, not MIN/MAX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authority name, bidder name, and authority type now use the most-frequent value per ЕИК with a deterministic tiebreak (mixed-case over ALL-CAPS via a Cyrillic-aware GLOB, then longer, then lexical), mirrored in normalize-raw.sql and refresh-slice.sql. Keys stay ЕИК-based so URLs are unaffected. Extracts the canonical-authority-name fix from #203 (@StanislavBG) and adds the two asymmetrically-missed siblings: bidder name (was MIN(contractor_name)) and authority type (was MAX(authority_type), which also mis-drove the state-company bucket). --- .../etl-entity-canonicalization-sql.test.ts | 183 ++++++++++++++++++ scripts/normalize-raw.sql | 145 ++++++++++++-- scripts/refresh-slice.sql | 156 +++++++++++++-- 3 files changed, 454 insertions(+), 30 deletions(-) create mode 100644 packages/db/src/etl-entity-canonicalization-sql.test.ts diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts new file mode 100644 index 000000000..a4029351b --- /dev/null +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -0,0 +1,183 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const stagingPath = resolve(root, 'scripts/work-staging-schema.sql'); +const etlPaths = [ + ['normalize-raw', resolve(root, 'scripts/normalize-raw.sql')], + ['refresh-slice', resolve(root, 'scripts/refresh-slice.sql')], +] as const; + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +function withEtlDb(label: string, run: (dbPath: string) => void): void { + const dir = mkdtempSync(resolve(tmpdir(), `sigma-entity-canonicalization-${label}-`)); + const dbPath = resolve(dir, 'test.sqlite'); + try { + readScript(dbPath, schemaPath); + readScript(dbPath, stagingPath); + run(dbPath); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +describe('ETL entity canonicalization through real SQL scripts', () => { + it('uses the authority-name mode and all deterministic tiebreakers in both ETL paths', () => { + const modeEik = '300000001'; + const caseEik = '300000002'; + const lengthEik = '300000003'; + const lexicalEik = '300000004'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type) + VALUES + ('eop:tenders:mode-1', '2026-06-01T00:00:00Z', 'UNP-MODE-1', '${modeEik}', + 'БСУ Д-р Петър Берон', 'public'), + ('eop:tenders:mode-2', '2026-06-01T00:00:00Z', 'UNP-MODE-2', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:mode-3', '2026-06-01T00:00:00Z', 'UNP-MODE-3', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:mode-4', '2026-06-01T00:00:00Z', 'UNP-MODE-4', '${modeEik}', + 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА', 'public'), + ('eop:tenders:case-1', '2026-06-01T00:00:00Z', 'UNP-CASE-1', '${caseEik}', + 'TEST AUTHORITY', 'public'), + ('eop:tenders:case-2', '2026-06-01T00:00:00Z', 'UNP-CASE-2', '${caseEik}', + 'Test Authority', 'public'), + ('eop:tenders:length-1', '2026-06-01T00:00:00Z', 'UNP-LENGTH-1', '${lengthEik}', + 'Short name', 'public'), + ('eop:tenders:length-2', '2026-06-01T00:00:00Z', 'UNP-LENGTH-2', '${lengthEik}', + 'Longer authority name', 'public'), + ('eop:tenders:lexical-1', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-1', '${lexicalEik}', + 'Alpha key', 'public'), + ('eop:tenders:lexical-2', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-2', '${lexicalEik}', + 'Bravo key', 'public');`, + ); + + expect( + sqliteJson<{ name: string }>( + dbPath, + `SELECT MIN(authority_name) AS name FROM raw_tenders WHERE authority_eik = '${modeEik}'`, + )[0]?.name, + ).toBe('БСУ Д-р Петър Берон'); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ bulstat: string; name: string }>( + dbPath, + `SELECT bulstat, name FROM authorities + WHERE bulstat IN ('${modeEik}', '${caseEik}', '${lengthEik}', '${lexicalEik}') + ORDER BY bulstat`, + ), + ).toEqual([ + { bulstat: modeEik, name: 'МИНИСТЕРСТВО НА ОБРАЗОВАНИЕТО И НАУКАТА' }, + { bulstat: caseEik, name: 'Test Authority' }, + { bulstat: lengthEik, name: 'Longer authority name' }, + { bulstat: lexicalEik, name: 'Alpha key' }, + ]); + }); + } + }); + + it('uses the bidder-name mode for EIK- and name-keyed bidders in both ETL paths', () => { + const bidderEik = '400000001'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_contracts + (source, fetched_at, contractor_eik, contractor_name) + VALUES + ('eop:contracts:bidder-1', '2026-06-01T00:00:00Z', '${bidderEik}', 'A Rare Bidder'), + ('eop:contracts:bidder-2', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:bidder-3', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:bidder-4', '2026-06-01T00:00:00Z', '${bidderEik}', 'Modal Bidder'), + ('eop:contracts:name-1', '2026-06-01T00:00:00Z', 'foreign-id', 'ACME'), + ('eop:contracts:name-2', '2026-06-01T00:00:00Z', 'foreign-id', 'Acme');`, + ); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ id: string; name: string }>( + dbPath, + `SELECT id, name FROM bidders + WHERE id IN ('eik:${bidderEik}', 'name:ACME') + ORDER BY id`, + ), + ).toEqual([ + { id: `eik:${bidderEik}`, name: 'Modal Bidder' }, + { id: 'name:ACME', name: 'Acme' }, + ]); + }); + } + }); + + it('uses the modal type for the winning authority name and buckets from raw types', () => { + const authorityEik = '500000001'; + + for (const [label, scriptPath] of etlPaths) { + withEtlDb(label, (dbPath) => { + sqlite( + dbPath, + `INSERT INTO raw_tenders + (source, fetched_at, unp, authority_eik, authority_name, authority_type) + VALUES + ('eop:tenders:type-1', '2026-06-01T00:00:00Z', 'UNP-TYPE-1', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-2', '2026-06-01T00:00:00Z', 'UNP-TYPE-2', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-3', '2026-06-01T00:00:00Z', 'UNP-TYPE-3', '${authorityEik}', + 'Independent authority', 'Орган на централната власт'), + ('eop:tenders:type-4', '2026-06-01T00:00:00Z', 'UNP-TYPE-4', '${authorityEik}', + 'Independent authority', 'Публично предприятие - сектор'), + ('eop:tenders:type-5', '2026-06-01T00:00:00Z', 'UNP-TYPE-5', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода'), + ('eop:tenders:type-6', '2026-06-01T00:00:00Z', 'UNP-TYPE-6', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода'), + ('eop:tenders:type-7', '2026-06-01T00:00:00Z', 'UNP-TYPE-7', '${authorityEik}', + 'A misleading label', 'Комунални услуги - вода');`, + ); + + readScript(dbPath, scriptPath); + + expect( + sqliteJson<{ name: string; type: string; type_group: string }>( + dbPath, + `SELECT name, type, type_group FROM authorities WHERE bulstat = '${authorityEik}'`, + )[0], + ).toEqual({ + name: 'Independent authority', + type: 'Орган на централната власт', + type_group: 'държавна компания', + }); + }); + } + }); +}); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 17cad66ed..97ba26759 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -46,14 +46,82 @@ DELETE FROM authorities; -- 1) Authorities — dedupe on ЕИК across both contracts and tenders staging, keep a -- canonical display name and the authority type (Вид на възложителя). -INSERT OR IGNORE INTO authorities (id, name, bulstat, type) -SELECT 'auth:' || authority_eik, MIN(authority_name), authority_eik, MAX(authority_type) +-- Names use the mode per ЕИК, with deterministic presentation-quality tiebreaks. +DROP TABLE IF EXISTS authority_canonical_name; +CREATE TABLE authority_canonical_name (authority_eik TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO authority_canonical_name (authority_eik, canonical_name) +SELECT authority_eik, authority_name +FROM ( + SELECT + authority_eik, + authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY + cnt DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, + authority_name + ) AS rn + FROM ( + SELECT authority_eik, authority_name, COUNT(*) AS cnt + FROM ( + SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL + UNION ALL + SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL + ) + WHERE authority_name IS NOT NULL AND TRIM(authority_name) <> '' + GROUP BY authority_eik, authority_name + ) +) +WHERE rn = 1; + +-- Keep the type modal within rows carrying the winning name, so the two canonical fields describe +-- the same entity when an ЕИК has been shared or reused under multiple labels. +DROP TABLE IF EXISTS authority_canonical_type; +CREATE TABLE authority_canonical_type (authority_eik TEXT PRIMARY KEY, canonical_type TEXT); +INSERT INTO authority_canonical_type (authority_eik, canonical_type) +SELECT authority_eik, authority_type FROM ( - SELECT authority_eik, authority_name, authority_type FROM raw_contracts WHERE authority_eik IS NOT NULL - UNION ALL - SELECT authority_eik, authority_name, authority_type FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT + authority_eik, + authority_type, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY cnt DESC, authority_type + ) AS rn + FROM ( + SELECT authority_eik, authority_type, COUNT(*) AS cnt + FROM ( + SELECT c.authority_eik, c.authority_type + FROM raw_contracts c + JOIN authority_canonical_name acn + ON acn.authority_eik = c.authority_eik AND acn.canonical_name = c.authority_name + UNION ALL + SELECT t.authority_eik, t.authority_type + FROM raw_tenders t + JOIN authority_canonical_name acn + ON acn.authority_eik = t.authority_eik AND acn.canonical_name = t.authority_name + ) + WHERE authority_type IS NOT NULL AND TRIM(authority_type) <> '' + GROUP BY authority_eik, authority_type + ) ) -GROUP BY authority_eik; +WHERE rn = 1; + +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT + 'auth:' || s.authority_eik, + COALESCE(acn.canonical_name, s.authority_eik), + s.authority_eik, + act.canonical_type +FROM ( + SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL + UNION + SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL +) s +LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik +LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; -- 1b) Friendly authority type buckets — heuristic from name + ЗОП type (non-critical display field; -- name patterns cover Title- and UPPER-case Cyrillic since SQLite LIKE is case-sensitive for it). @@ -63,7 +131,19 @@ UPDATE authorities SET type_group = CASE WHEN name LIKE '%болница%' OR name LIKE '%БОЛНИЦА%' OR name LIKE 'МБАЛ%' OR name LIKE '%МБАЛ%' OR name LIKE '%СБАЛ%' OR name LIKE '%ДКЦ%' OR name LIKE '%лечебно заведение%' THEN 'болница' WHEN name LIKE '%университет%' OR name LIKE '%УНИВЕРСИТЕТ%' OR name LIKE '%училище%' OR name LIKE '%УЧИЛИЩЕ%' OR name LIKE '%гимназия%' OR name LIKE '%ГИМНАЗИЯ%' OR name LIKE '%детска градина%' OR name LIKE '%ДЕТСКА ГРАДИНА%' OR name LIKE '%академия%' THEN 'образование' WHEN name LIKE '%агенция%' OR name LIKE '%Агенция%' OR name LIKE '%АГЕНЦИЯ%' THEN 'агенция' - WHEN type LIKE 'Публично предприятие%' OR type LIKE 'Комунални услуги%' THEN 'държавна компания' + WHEN EXISTS ( + SELECT 1 + FROM ( + SELECT authority_eik, authority_type FROM raw_contracts + UNION ALL + SELECT authority_eik, authority_type FROM raw_tenders + ) raw_authority_types + WHERE raw_authority_types.authority_eik = authorities.bulstat + AND ( + raw_authority_types.authority_type LIKE 'Публично предприятие%' + OR raw_authority_types.authority_type LIKE 'Комунални услуги%' + ) + ) THEN 'държавна компания' ELSE 'друго' END; @@ -238,10 +318,50 @@ WHERE t.lot_id IS NOT NULL -- which keeps the bulstat UNIQUE happy). is_consortium describes the ENTITY (a JV), so it is -- name-based — a semicolon member list or ДЗЗД / ОБЕДИНЕНИЕ / КОНСОРЦИУМ in the name; the -- per-contract awarded_to_group flag lives on contracts, not here. +DROP TABLE IF EXISTS bidder_canonical_name; +CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO bidder_canonical_name (bidder_key, canonical_name) +SELECT bidder_key, contractor_name +FROM ( + SELECT + bidder_key, + contractor_name, + ROW_NUMBER() OVER ( + PARTITION BY bidder_key + ORDER BY + cnt DESC, + CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(contractor_name) DESC, + contractor_name + ) AS rn + FROM ( + SELECT bidder_key, contractor_name, COUNT(*) AS cnt + FROM ( + SELECT + contractor_name, + CASE + WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END AS bidder_key + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) + ) + WHERE bidder_key IS NOT NULL + AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' + GROUP BY bidder_key, contractor_name + ) +) +WHERE rn = 1; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - bidder_key, - MIN(contractor_name), + b.bidder_key, + COALESCE(bcn.canonical_name, b.bidder_key), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), @@ -270,9 +390,10 @@ FROM ( TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' ) -) -WHERE bidder_key IS NOT NULL -GROUP BY bidder_key; +) b +LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key +WHERE b.bidder_key IS NOT NULL +GROUP BY b.bidder_key; -- 4b) Curated public-owned winner classification. Exact EIK matches cover the allowlist; a small -- branch list handles valid 13-digit branch EIKs used by AПИ/ОПУ, ЕСО/МЕР, БНР and Информационно diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed2..c31f99620 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -22,15 +22,85 @@ CREATE TABLE refresh_touched_authorities (authority_id TEXT PRIMARY KEY); -- @refresh-batch authorities-bidders -- ── 1) Authorities referenced by OCDS staging (new ones only; INSERT OR IGNORE) ──────────────────── -INSERT OR IGNORE INTO authorities (id, name, bulstat, type) -SELECT 'auth:' || authority_eik, MIN(authority_name), authority_eik, MAX(authority_type) +DROP TABLE IF EXISTS authority_canonical_name; +CREATE TABLE authority_canonical_name (authority_eik TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO authority_canonical_name (authority_eik, canonical_name) +SELECT authority_eik, authority_name FROM ( - SELECT source, authority_eik, authority_name, authority_type FROM raw_contracts - UNION ALL - SELECT source, authority_eik, authority_name, authority_type FROM raw_tenders + SELECT + authority_eik, + authority_name, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY + cnt DESC, + CASE WHEN authority_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(authority_name) DESC, + authority_name + ) AS rn + FROM ( + SELECT authority_eik, authority_name, COUNT(*) AS cnt + FROM ( + SELECT source, authority_eik, authority_name FROM raw_contracts + UNION ALL + SELECT source, authority_eik, authority_name FROM raw_tenders + ) + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') + AND authority_eik IS NOT NULL + AND authority_name IS NOT NULL AND TRIM(authority_name) <> '' + GROUP BY authority_eik, authority_name + ) ) -WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL -GROUP BY authority_eik; +WHERE rn = 1; + +DROP TABLE IF EXISTS authority_canonical_type; +CREATE TABLE authority_canonical_type (authority_eik TEXT PRIMARY KEY, canonical_type TEXT); +INSERT INTO authority_canonical_type (authority_eik, canonical_type) +SELECT authority_eik, authority_type +FROM ( + SELECT + authority_eik, + authority_type, + ROW_NUMBER() OVER ( + PARTITION BY authority_eik + ORDER BY cnt DESC, authority_type + ) AS rn + FROM ( + SELECT authority_eik, authority_type, COUNT(*) AS cnt + FROM ( + SELECT c.authority_eik, c.authority_type + FROM raw_contracts c + JOIN authority_canonical_name acn + ON acn.authority_eik = c.authority_eik AND acn.canonical_name = c.authority_name + WHERE c.source LIKE 'eop:%' OR c.source LIKE 'ocds:%' + UNION ALL + SELECT t.authority_eik, t.authority_type + FROM raw_tenders t + JOIN authority_canonical_name acn + ON acn.authority_eik = t.authority_eik AND acn.canonical_name = t.authority_name + WHERE t.source LIKE 'eop:%' OR t.source LIKE 'ocds:%' + ) + WHERE authority_type IS NOT NULL AND TRIM(authority_type) <> '' + GROUP BY authority_eik, authority_type + ) +) +WHERE rn = 1; + +INSERT OR IGNORE INTO authorities (id, name, bulstat, type) +SELECT + 'auth:' || s.authority_eik, + COALESCE(acn.canonical_name, s.authority_eik), + s.authority_eik, + act.canonical_type +FROM ( + SELECT authority_eik FROM raw_contracts + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + UNION + SELECT authority_eik FROM raw_tenders + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL +) s +LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik +LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; -- type_group for any authority still missing it (covers the rows just inserted) — same heuristic as -- normalize-raw.sql step 1b. @@ -40,16 +110,69 @@ UPDATE authorities SET type_group = CASE WHEN name LIKE '%болница%' OR name LIKE '%БОЛНИЦА%' OR name LIKE 'МБАЛ%' OR name LIKE '%МБАЛ%' OR name LIKE '%СБАЛ%' OR name LIKE '%ДКЦ%' OR name LIKE '%лечебно заведение%' THEN 'болница' WHEN name LIKE '%университет%' OR name LIKE '%УНИВЕРСИТЕТ%' OR name LIKE '%училище%' OR name LIKE '%УЧИЛИЩЕ%' OR name LIKE '%гимназия%' OR name LIKE '%ГИМНАЗИЯ%' OR name LIKE '%детска градина%' OR name LIKE '%ДЕТСКА ГРАДИНА%' OR name LIKE '%академия%' THEN 'образование' WHEN name LIKE '%агенция%' OR name LIKE '%Агенция%' OR name LIKE '%АГЕНЦИЯ%' THEN 'агенция' - WHEN type LIKE 'Публично предприятие%' OR type LIKE 'Комунални услуги%' THEN 'държавна компания' + WHEN EXISTS ( + SELECT 1 + FROM ( + SELECT source, authority_eik, authority_type FROM raw_contracts + UNION ALL + SELECT source, authority_eik, authority_type FROM raw_tenders + ) raw_authority_types + WHERE (raw_authority_types.source LIKE 'eop:%' OR raw_authority_types.source LIKE 'ocds:%') + AND raw_authority_types.authority_eik = authorities.bulstat + AND ( + raw_authority_types.authority_type LIKE 'Публично предприятие%' + OR raw_authority_types.authority_type LIKE 'Комунални услуги%' + ) + ) THEN 'държавна компания' ELSE 'друго' END WHERE type_group IS NULL; -- ── 2) Bidders referenced by OCDS staging (new ones only) — same identity rule as normalize step 4 ── +DROP TABLE IF EXISTS bidder_canonical_name; +CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); +INSERT INTO bidder_canonical_name (bidder_key, canonical_name) +SELECT bidder_key, contractor_name +FROM ( + SELECT + bidder_key, + contractor_name, + ROW_NUMBER() OVER ( + PARTITION BY bidder_key + ORDER BY + cnt DESC, + CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(contractor_name) DESC, + contractor_name + ) AS rn + FROM ( + SELECT bidder_key, contractor_name, COUNT(*) AS cnt + FROM ( + SELECT + contractor_name, + CASE + WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean + WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) + ELSE NULL + END AS bidder_key + FROM ( + SELECT + contractor_name, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + ) + ) + WHERE bidder_key IS NOT NULL + AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' + GROUP BY bidder_key, contractor_name + ) +) +WHERE rn = 1; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - bidder_key, - MIN(contractor_name), + b.bidder_key, + COALESCE(bcn.canonical_name, b.bidder_key), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), @@ -75,15 +198,12 @@ FROM ( TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' ) -) -WHERE bidder_key IS NOT NULL -GROUP BY bidder_key +) b +LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key +WHERE b.bidder_key IS NOT NULL +GROUP BY b.bidder_key ON CONFLICT(id) DO UPDATE SET - name = CASE - WHEN excluded.name IS NULL THEN bidders.name - WHEN bidders.name IS NULL THEN excluded.name - ELSE min(bidders.name, excluded.name) - END, + name = excluded.name, bulstat = COALESCE(bidders.bulstat, excluded.bulstat), eik_normalized = COALESCE(bidders.eik_normalized, excluded.eik_normalized), eik_valid = max(bidders.eik_valid, excluded.eik_valid), From 45b100a8c594aa3bb72dd539100c3255601bc9b8 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Fri, 17 Jul 2026 22:00:28 +0000 Subject: [PATCH 3/9] feat(etl,web): preserve verbatim ordering-unit name beside the canonical authority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under #194 the authority display name is canonicalized to the frequency mode per ЕИК, which collapses genuine sub-units that share one ЕИК (e.g. a school placed under МОН's ЕИК keeps the ministry name). Additionally keep the raw per-row "Възложител" verbatim so no ordering unit is lost. - add contracts.ordering_unit_name / tenders.ordering_unit_name, filled from the raw authority_name in normalize-raw and both refresh-slice paths - getContract surfaces it on the contract page only when it differs from the canonical name after a Cyrillic-safe fold (case + whitespace), so spelling variants stay hidden and real sub-units show - /contracts/:id.json sourceNames.authority is now genuinely verbatim; it previously served the canonical name despite the "verbatim" contract Extends #251; credit StanislavBG's #203 identity work. --- apps/web/app/routes/contract.tsx | 5 +++ packages/api-contract/src/index.ts | 1 + packages/db/migrations/0000_init.sql | 2 ++ .../etl-entity-canonicalization-sql.test.ts | 33 +++++++++++++++++-- packages/db/src/queries/details.ts | 17 +++++++++- scripts/normalize-raw.sql | 7 ++-- scripts/refresh-slice.sql | 12 +++++-- 7 files changed, 69 insertions(+), 8 deletions(-) diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index 256617de4..ce7855af8 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -269,6 +269,11 @@ export default function Contract({ loaderData }: Route.ComponentProps) {

{c.authority.name}

+ {c.authority.orderingUnit && ( +

+ Възложител по документа: {c.authority.orderingUnit} +

+ )}

{c.authority.typeLabel && {c.authority.typeLabel}} {c.authority.settlement && <> {c.authority.settlement}} diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index ad7d6d1d7..4d8408676 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -235,6 +235,7 @@ export interface ContractParty { slug: string; name: string; displayName: string; + orderingUnit: string | null; kind?: EntityKind; typeLabel: string | null; settlement: string | null; diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 0888ef93d..51127e8a8 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -42,6 +42,7 @@ CREATE TABLE tenders ( source_id TEXT NOT NULL UNIQUE, -- УНП (ЦАИС ЕОП identifier) title TEXT NOT NULL, authority_id TEXT NOT NULL REFERENCES authorities(id), + ordering_unit_name TEXT, -- verbatim "Възложител" (raw authority_name) on this row; preserves sub-unit identity collapsed by canonical authorities.name (mode) cpv_code TEXT, cpv_description TEXT, -- human-readable CPV label (no external dictionary needed) estimated_value REAL, @@ -112,6 +113,7 @@ CREATE TABLE contracts ( id TEXT PRIMARY KEY, -- 'c:' || staging row id tender_id TEXT NOT NULL REFERENCES tenders(id), bidder_id TEXT NOT NULL REFERENCES bidders(id), + ordering_unit_name TEXT, -- verbatim "Възложител" (raw authority_name) on this row; preserves sub-unit identity collapsed by canonical authorities.name (mode) amount REAL NOT NULL, -- as-recorded headline value in `currency` (display); signing for annex_suspect currency TEXT NOT NULL DEFAULT 'BGN', signed_at TEXT, diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index a4029351b..ca85461b5 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -44,7 +44,7 @@ function withEtlDb(label: string, run: (dbPath: string) => void): void { describe('ETL entity canonicalization through real SQL scripts', () => { it('uses the authority-name mode and all deterministic tiebreakers in both ETL paths', () => { - const modeEik = '300000001'; + const modeEik = '000695114'; const caseEik = '300000002'; const lengthEik = '300000003'; const lexicalEik = '300000004'; @@ -75,7 +75,18 @@ describe('ETL entity canonicalization through real SQL scripts', () => { ('eop:tenders:lexical-1', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-1', '${lexicalEik}', 'Alpha key', 'public'), ('eop:tenders:lexical-2', '2026-06-01T00:00:00Z', 'UNP-LEXICAL-2', '${lexicalEik}', - 'Bravo key', 'public');`, + 'Bravo key', 'public'); + + INSERT INTO raw_contracts + (source, fetched_at, unp, authority_eik, authority_name, contract_number, + contract_date, signing_value, currency, contractor_eik, contractor_name) + VALUES + ('eop:contracts:subunit', '2026-06-01T00:00:00Z', 'UNP-MODE-1', '${modeEik}', + 'СУ „Христо Ботев" гр. Пловдив', 'CONTRACT-SUBUNIT', '2026-06-01', 1000, 'BGN', + '400000011', 'Subunit Bidder'), + ('eop:contracts:case', '2026-06-01T00:00:00Z', 'UNP-MODE-2', '${modeEik}', + 'Министерство на образованието и науката', 'CONTRACT-CASE', '2026-06-01', 1000, + 'BGN', '400000012', 'Case Bidder');`, ); expect( @@ -100,6 +111,24 @@ describe('ETL entity canonicalization through real SQL scripts', () => { { bulstat: lengthEik, name: 'Longer authority name' }, { bulstat: lexicalEik, name: 'Alpha key' }, ]); + + expect( + sqliteJson<{ contract_number: string; ordering_unit_name: string }>( + dbPath, + `SELECT contract_number, ordering_unit_name FROM contracts + WHERE contract_number IN ('CONTRACT-SUBUNIT', 'CONTRACT-CASE') + ORDER BY contract_number`, + ), + ).toEqual([ + { + contract_number: 'CONTRACT-CASE', + ordering_unit_name: 'Министерство на образованието и науката', + }, + { + contract_number: 'CONTRACT-SUBUNIT', + ordering_unit_name: 'СУ „Христо Ботев" гр. Пловдив', + }, + ]); }); } }); diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5e..71f8347e3 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -362,6 +362,8 @@ export async function getAuthority( // ── Contract ────────────────────────────────────────────────────────────────────────────────── +const foldName = (s: string): string => s.toLocaleLowerCase('bg').replace(/\s+/g, ' ').trim(); + interface ContractDetailRow { id: string; tender_id: string; @@ -408,6 +410,7 @@ interface ContractDetailRow { // authority authority_id: string; authority_name: string; + source_authority_name: string | null; authority_type_group: string | null; authority_settlement: string | null; // bidder @@ -466,6 +469,7 @@ export async function getContract( c.signing_value_eur, c.current_value_eur, c.value_flag, c.date_flag, c.bids_received, c.bids_rejected, c.bids_sme, c.bids_non_eea, c.subcontractor_eik, c.subcontractor_name, c.subcontract_value, c.currency AS contract_currency, + c.ordering_unit_name AS source_authority_name, t.title, t.source_id AS unp, t.procedure_type, t.cpv_code, t.cpv_description, t.num_lots, t.eop_tender_id, t.estimated_value, t.currency AS tender_currency, t.start_date, t.end_date, @@ -602,6 +606,10 @@ export async function getContract( slug: authoritySlug(r.authority_id), name: cleanName(r.authority_name), displayName: cleanName(r.authority_name), + orderingUnit: + r.source_authority_name && foldName(r.source_authority_name) !== foldName(r.authority_name) + ? cleanName(r.source_authority_name) + : null, typeLabel: typeLabel(r.authority_type_group), settlement: r.authority_settlement, eik: authoritySlug(r.authority_id), @@ -613,6 +621,7 @@ export async function getContract( slug: companySlug(r.bidder_id), name: cleanName(r.bidder_name), displayName: entityName(cleanName(r.bidder_name), r.bidder_kind), + orderingUnit: null, kind: r.bidder_kind, typeLabel: null, settlement: r.bidder_settlement, @@ -696,5 +705,11 @@ export async function getContract( amendments, }; - return { ...detail, sourceNames: { authority: r.authority_name, bidder: r.bidder_name } }; + return { + ...detail, + sourceNames: { + authority: r.source_authority_name ?? r.authority_name, + bidder: r.bidder_name, + }, + }; } diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 97ba26759..0b4f1018d 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -150,7 +150,7 @@ END; -- 2a) Tenders — the header row of each procurement (lot_id IS NULL): one per УНП, carrying -- procedure type, CPV, the procurement-level estimated value, lot count and authority. INSERT OR IGNORE INTO tenders - (id, source_id, title, authority_id, cpv_code, cpv_description, estimated_value, currency, + (id, source_id, title, authority_id, ordering_unit_name, cpv_code, cpv_description, estimated_value, currency, procedure_type, contract_kind, num_lots, status, published_at, deadline_at, legal_basis, award_criteria, main_activity, notice_type, place_of_performance, start_date, end_date, duration, duration_unit, @@ -160,6 +160,7 @@ SELECT t.unp, COALESCE(t.procurement_subject, '(без предмет)'), 'auth:' || t.authority_eik, + NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, t.estimated_value, @@ -443,7 +444,7 @@ SET ownership_kind = ( -- fx_rate carries the applied rate on the row (amount * fx_rate = amount_eur), so the original value, -- the rate, and the EUR value are all auditable without joining fx_rates. INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, lot_id, document_number, published_at, contract_subject, @@ -461,6 +462,7 @@ SELECT END, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -619,6 +621,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index c31f99620..a79a2ea7f 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -258,7 +258,7 @@ WHERE c.id IN (SELECT id FROM refresh_touched_contracts) -- @refresh-batch tenders -- EOP tender headers and lots loaded since the last full normalize. INSERT INTO tenders - (id, source_id, title, authority_id, cpv_code, cpv_description, estimated_value, currency, + (id, source_id, title, authority_id, ordering_unit_name, cpv_code, cpv_description, estimated_value, currency, procedure_type, contract_kind, num_lots, status, published_at, deadline_at, legal_basis, award_criteria, main_activity, notice_type, place_of_performance, start_date, end_date, duration, duration_unit, @@ -268,6 +268,7 @@ SELECT t.unp, COALESCE(t.procurement_subject, '(без предмет)'), 'auth:' || t.authority_eik, + NULLIF(TRIM(t.authority_name), ''), t.cpv_code, t.cpv_description, t.estimated_value, @@ -301,6 +302,7 @@ ON CONFLICT(id) DO UPDATE SET source_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.source_id ELSE tenders.source_id END, title = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.title ELSE tenders.title END, authority_id = CASE WHEN tenders.procedure_type = 'неизвестна' THEN excluded.authority_id ELSE tenders.authority_id END, + ordering_unit_name = excluded.ordering_unit_name, cpv_code = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.cpv_code, tenders.cpv_code) ELSE tenders.cpv_code END, cpv_description = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.cpv_description, tenders.cpv_description) ELSE tenders.cpv_description END, estimated_value = CASE WHEN tenders.procedure_type = 'неизвестна' THEN COALESCE(excluded.estimated_value, tenders.estimated_value) ELSE tenders.estimated_value END, @@ -635,7 +637,7 @@ WHERE id IN ( ) ); INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -648,6 +650,7 @@ SELECT COALESCE(NULLIF(x.lot_id, ''), '_') || ':' || x.bidder_key || ':' || x.contract_ordinal, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -808,6 +811,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -879,7 +883,7 @@ WHERE id IN ( ); INSERT OR IGNORE INTO contracts - (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, + (id, tender_id, bidder_id, ordering_unit_name, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, eu_funded, bids_received, contract_kind, awarded_to_group, value_flag, date_flag, amount_eur, fx_converted, fx_rate, signing_value_eur, current_value_eur, lot_id, document_number, published_at, contract_subject, @@ -892,6 +896,7 @@ SELECT COALESCE(NULLIF(x.lot_norm, ''), '_') || ':' || x.bidder_key || ':' || x.contract_ordinal, 't:' || x.unp, x.bidder_key, + NULLIF(TRIM(x.authority_name_raw), ''), x.display_native, COALESCE(x.currency, 'BGN'), x.contract_date, @@ -1056,6 +1061,7 @@ FROM ( END AS bidder_key FROM ( SELECT c.*, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 From 897a4213f0b5e1b48739c9b19272957d23e85e09 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sat, 18 Jul 2026 14:15:21 +0000 Subject: [PATCH 4/9] fix(etl,web): preserve identity-poor contracts via an unknown-bidder bucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bidder key fell to NULL when a contractor had an invalid-checksum ЕИК AND no name; `WHERE bidder_key IS NOT NULL` then silently dropped the whole contract. Never drop a contract for identity reasons — downgrade the key. - add a third key rung: invalid ЕИК + empty/NULL name -> one labelled `unknown:анонимен` bucket ("Неизвестен изпълнител", kind=unknown), so every previously-dropped contract stays in the corpus - compute the checksum + the three-rung key ONCE in a scratch `contractor_identity` table (NULL-safe raw-pair join), replacing the ~7 copy-pasted checksum blocks; delete the consistency test that pinned them - keep the bucket inside company_totals so money still reconciles; hide it only from browsable rankings (search_index, /companies, home top suppliers) - resolve the bucket's /companies slug so the contract-page link works - mirror all of it in refresh-slice; the checksum algorithm is byte-identical A new SQL test asserts no eligible contract is dropped and SUM(amount_eur) is preserved across both the normalize and refresh paths. --- .../db/src/contractor-identity-sql.test.ts | 104 +++++++ .../db/src/eik-checksum-consistency.test.ts | 33 --- packages/db/src/queries/companies.test.ts | 20 ++ packages/db/src/queries/companies.ts | 2 +- packages/db/src/queries/home.test.ts | 12 +- packages/db/src/queries/home.ts | 4 +- packages/db/src/queries/identity.test.ts | 4 + packages/db/src/queries/identity.ts | 4 +- scripts/normalize-raw.sql | 245 +++++++--------- scripts/precompute.sql | 3 +- scripts/refresh-slice.sql | 266 +++++++----------- 11 files changed, 347 insertions(+), 350 deletions(-) create mode 100644 packages/db/src/contractor-identity-sql.test.ts delete mode 100644 packages/db/src/eik-checksum-consistency.test.ts diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts new file mode 100644 index 000000000..1bcfbb16b --- /dev/null +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -0,0 +1,104 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schema = readFileSync(resolve(root, 'packages/db/migrations/0000_init.sql'), 'utf8'); +const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); +const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); +const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); +const refresh = readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8'); + +const seed = ` +INSERT INTO raw_contracts + (source, fetched_at, unp, contract_number, contract_date, published_at, + authority_eik, authority_name, procurement_subject, estimated_value, + signing_value, currency, contractor_eik, contractor_name) +VALUES + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-VALID', 'C-VALID', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Валиден ЕИК', 100, 100, 'EUR', + '103267194', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-TYPO', 'C-TYPO', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Грешна контролна цифра', 200, 200, 'EUR', + '103267195', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-EMPTY', 'C-EMPTY', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Празно име', 300, 300, 'EUR', + '103267195', ''), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-NULL', 'C-NULL', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Липсваща самоличност', 400, 400, 'EUR', + NULL, NULL); +`; + +function build(path: 'normalize' | 'refresh'): DatabaseSync { + const db = new DatabaseSync(':memory:'); + db.exec(schema); + db.exec(staging); + db.exec(seed); + if (path === 'normalize') { + db.exec(normalize); + db.exec(precompute); + } else { + db.exec(refresh); + } + return db; +} + +describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path) => { + it('preserves every eligible contract across valid, typo, empty, and NULL identities', () => { + const db = build(path); + try { + const contracts = db + .prepare('SELECT contract_number, bidder_id FROM contracts ORDER BY contract_number') + .all() as { contract_number: string; bidder_id: string }[]; + + expect(contracts).toEqual([ + { contract_number: 'C-EMPTY', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-NULL', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-TYPO', bidder_id: 'name:ТИПО ООД' }, + { contract_number: 'C-VALID', bidder_id: 'eik:103267194' }, + ]); + + expect( + db + .prepare( + `SELECT id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind + FROM bidders WHERE id = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ + id: 'unknown:анонимен', + name: 'Неизвестен изпълнител', + bulstat: null, + eik_normalized: null, + eik_valid: 0, + is_consortium: 0, + kind: 'unknown', + }); + + const sums = db + .prepare( + `SELECT + (SELECT SUM(signing_value) FROM raw_contracts + WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') AS raw_total, + (SELECT SUM(amount_eur) FROM contracts) AS contract_total, + (SELECT SUM(won_eur) FROM company_totals) AS company_total`, + ) + .get() as { raw_total: number; contract_total: number; company_total: number }; + expect(sums).toEqual({ raw_total: 1000, contract_total: 1000, company_total: 1000 }); + + expect( + db + .prepare( + `SELECT COUNT(*) AS n FROM search_index + WHERE kind = 'company' AND ref = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ n: 0 }); + } finally { + db.close(); + } + }); +}); diff --git a/packages/db/src/eik-checksum-consistency.test.ts b/packages/db/src/eik-checksum-consistency.test.ts deleted file mode 100644 index 606cf4e03..000000000 --- a/packages/db/src/eik-checksum-consistency.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -/// -import { readFileSync } from 'node:fs'; -import { dirname, resolve } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { describe, expect, it } from 'vitest'; - -const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); - -function checksumBlocks(path: string): string[] { - const sql = readFileSync(path, 'utf8'); - const pattern = - /^[ \t]*CASE\r?\n(?=[ \t]+WHEN eik_clean IS NULL\b[^\r\n]*LENGTH\(eik_clean\) NOT IN \(9, 13\) THEN 0$)[\s\S]*?^[ \t]*END AS eik_valid$/gm; - - return [...sql.matchAll(pattern)].map(([block]) => { - const lines = block.split(/\r?\n/); - const indentation = Math.min( - ...lines.filter((line) => line.trim()).map((line) => line.match(/^[ \t]*/)?.[0].length ?? 0), - ); - return lines.map((line) => line.slice(indentation)).join('\n'); - }); -} - -describe('ЕИК checksum copies stay consistent', () => { - it('keeps every checksum CASE block textually identical', () => { - const normalizeBlocks = checksumBlocks(resolve(root, 'scripts/normalize-raw.sql')); - const refreshBlocks = checksumBlocks(resolve(root, 'scripts/refresh-slice.sql')); - const blocks = [...normalizeBlocks, ...refreshBlocks]; - - expect(normalizeBlocks).toHaveLength(3); - expect(refreshBlocks).toHaveLength(4); - expect(new Set(blocks)).toEqual(new Set([blocks[0]])); - }); -}); diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index a4643d026..b0e9b9f0e 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -105,6 +105,26 @@ describe('streamCompaniesCsv', () => { expect(countRows(filtered)).toBeLessThan(countRows(unfiltered)); expect(countRows(filtered)).toBe(1); }); + + it('excludes the unknown identity bucket from the list and CSV', async () => { + const db = fakeDb(); + const sql: string[] = []; + const real = db.prepare.bind(db); + db.prepare = ((query: string) => { + sql.push(query); + return real(query); + }) as typeof db.prepare; + + await listCompanies(db, {}); + await streamCompaniesCsv(db, {}).text(); + + expect(sql.filter((query) => query.includes('company_totals'))).not.toHaveLength(0); + expect( + sql.filter((query) => query.includes('company_totals')).every((query) => + query.includes("kind <> 'unknown'"), + ), + ).toBe(true); + }); }); describe('prototype-key params (untrusted query values)', () => { diff --git a/packages/db/src/queries/companies.ts b/packages/db/src/queries/companies.ts index 70a1fa401..8f2665238 100644 --- a/packages/db/src/queries/companies.ts +++ b/packages/db/src/queries/companies.ts @@ -109,7 +109,7 @@ function source(p: CompanyListParams): { from: string; params: unknown[] } { * Keep consumed filter keys in sync with COMPANY_FILTER_KEYS and companyFilterSignature(). */ function entityWhere(p: CompanyListParams): { sql: string; params: unknown[] } { - const where: string[] = []; + const where: string[] = [`kind <> 'unknown'`]; const params: unknown[] = []; if (p.kinds?.length && p.kinds.length < 2) { where.push(`kind = ?`); diff --git a/packages/db/src/queries/home.test.ts b/packages/db/src/queries/home.test.ts index 0302eb457..257c8be50 100644 --- a/packages/db/src/queries/home.test.ts +++ b/packages/db/src/queries/home.test.ts @@ -61,9 +61,10 @@ const totalsRow = { refreshed_at: '2024-06-02T10:00:00Z', }; -function fakeDb(totals: typeof totalsRow | null): D1Database { +function fakeDb(totals: typeof totalsRow | null, capture?: string[]): D1Database { return { prepare(sql: string) { + capture?.push(sql); return { bind() { return this; @@ -121,6 +122,15 @@ describe('getHomeData', () => { expect(data.topMunicipalities).toHaveLength(1); }); + it('excludes the unknown identity bucket from top companies', async () => { + const sql: string[] = []; + await getHomeData(fakeDb(totalsRow, sql)); + + expect(sql.find((query) => query.includes('FROM company_totals'))).toContain( + "WHERE kind <> 'unknown'", + ); + }); + it('includes single-offer contract lists', async () => { const data = await getHomeData(fakeDb(totalsRow)); diff --git a/packages/db/src/queries/home.ts b/packages/db/src/queries/home.ts index fe588134a..41fc3ff3d 100644 --- a/packages/db/src/queries/home.ts +++ b/packages/db/src/queries/home.ts @@ -53,7 +53,9 @@ export async function getHomeData(db: D1Database): Promise { const [companies, ministries, municipalities, recentSingleOffer, topSingleOffer, singleOfferRow] = await Promise.all([ db - .prepare(`SELECT * FROM company_totals ORDER BY won_eur DESC, bidder_id LIMIT 10`) + .prepare( + `SELECT * FROM company_totals WHERE kind <> 'unknown' ORDER BY won_eur DESC, bidder_id LIMIT 10`, + ) .all(), db .prepare( diff --git a/packages/db/src/queries/identity.test.ts b/packages/db/src/queries/identity.test.ts index 2acc6ccfb..7ca09ed7b 100644 --- a/packages/db/src/queries/identity.test.ts +++ b/packages/db/src/queries/identity.test.ts @@ -25,6 +25,10 @@ describe('company slug', () => { it('round-trips a 13-digit ЕИК', () => { expect(bidderIdFromSlug(companySlug('eik:8316417910124'))).toBe('eik:8316417910124'); }); + it('resolves the unknown bidder linked from contract pages', () => { + expect(companySlug('unknown:анонимен')).toBe('unknown:анонимен'); + expect(bidderIdFromSlug('unknown:анонимен')).toBe('unknown:анонимен'); + }); }); describe('authority / contract slugs', () => { diff --git a/packages/db/src/queries/identity.ts b/packages/db/src/queries/identity.ts index a12067b33..2780bfc53 100644 --- a/packages/db/src/queries/identity.ts +++ b/packages/db/src/queries/identity.ts @@ -21,7 +21,8 @@ function b64urlDecode(s: string): string { const EIK_RE = /^\d{9}(\d{4})?$/; -/** bidder id → `/companies/:slug` segment. Valid ЕИК → the digits; name-keyed → `n` + base64url(name). */ +/** bidder id → `/companies/:slug` segment. Valid ЕИК → digits; name-keyed → `n` + base64url(name); + * the labelled unknown bucket stays literal so contract-page links remain resolvable. */ export function companySlug(bidderId: string): string { if (bidderId.startsWith('eik:')) return bidderId.slice(4); if (bidderId.startsWith('name:')) return 'n' + b64urlEncode(bidderId.slice(5)); @@ -31,6 +32,7 @@ export function companySlug(bidderId: string): string { /** `/companies/:slug` segment → bidder id, or null if it cannot be decoded. */ export function bidderIdFromSlug(slug: string): string | null { if (EIK_RE.test(slug)) return 'eik:' + slug; + if (slug === 'unknown:анонимен') return slug; if (slug.startsWith('n')) { try { return 'name:' + b64urlDecode(slug.slice(1)); diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 74ce5bf4f..c72a5f2a9 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -22,8 +22,8 @@ -- * Currency is kept per row as it appears (BGN pre-2026, EUR from 2026, a few foreign) — -- NOT coerced to one currency. Money sums must group/convert by currency downstream. -- * Authorities dedupe on ЕИК (Вид на възложителя kept as `type`); a canonical name is kept. --- * Bidders dedupe on raw contractor ЕИК (kept verbatim in bulstat); is_consortium comes from --- the admin "възложена на група" flag (awarded_to_group), not a name heuristic. Resolving the +-- * Bidders use a checksum-valid contractor ЕИК, then normalised name, then one unknown bucket; +-- is_consortium comes from the entity name, while awarded_to_group stays contract-scoped. Resolving the -- members hidden behind a single consortium ЕИК needs the Търговски регистър (joined on ЕИК), -- a parked future pipeline; for now the full value is attributed to the consortium entity. -- * Tenders come from the tenders-export header row (one per УНП); lots from its lot rows. @@ -229,83 +229,100 @@ FROM raw_tenders t WHERE t.lot_id IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || t.unp); --- 4) Bidders — winning contractors, with a robust identity key: --- * checksum-valid Bulgarian ЕИК → key by ЕИК ('eik:') --- * otherwise (withheld 'не се публикува', foreign id, multi-ЕИК consortium, missing) → --- key by NORMALISED NAME ('name:') --- Keying invalid ЕИК by name stops the collapse where ~595 distinct withheld-ЕИК contractors --- merged onto one node. bulstat/eik_normalized stay set only for a valid ЕИК (NULL otherwise, --- which keeps the bulstat UNIQUE happy). is_consortium describes the ENTITY (a JV), so it is --- name-based — a semicolon member list or ДЗЗД / ОБЕДИНЕНИЕ / КОНСОРЦИУМ in the name; the --- per-contract awarded_to_group flag lives on contracts, not here. -INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) +-- 4) Winning-contractor identity — derive the checksum and three-rung key exactly once, then use +-- the NULL-safe raw pair for every bidder/contract projection below. +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) SELECT - bidder_key, - MIN(contractor_name), - MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), - MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), - MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END + eik_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) + ELSE 'unknown:анонимен' + END FROM ( SELECT - contractor_name, + eik_raw, + name_raw, eik_clean, - eik_valid, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' - THEN 1 ELSE 0 - END AS grp + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid FROM ( SELECT - contractor_name, - eik_clean, - -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit - -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing - -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a - -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes - -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. - -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by - -- 3..10; a second 10 → 0. The 9th digit must equal the control. - -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight - -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit - -- must equal that control. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT - contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') ) -) -WHERE bidder_key IS NOT NULL +); + +-- 4a) Bidders — valid ЕИК first, then normalised name, then one labelled unknown bucket. The +-- bucket keeps identity-poor contracts attached without polluting bulstat/eik_normalized. +INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) +SELECT + bidder_key, + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' ELSE MIN(name_raw) END, + MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), + MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), + MAX(eik_valid), + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity GROUP BY bidder_key; -- 4b) Curated public-owned winner classification. Exact EIK matches cover the allowlist; a small @@ -523,42 +540,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -588,6 +572,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- EOP always; an OCDS row only when no EOP row shares its contract_number - EOP wins. @@ -616,8 +602,7 @@ FROM ( ) z ) y ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); @@ -818,42 +803,9 @@ SELECT 1, WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -883,6 +835,8 @@ SELECT 1, END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- Eligibility must mirror the INSERT INTO contracts WHERE exactly, so this candidate count is a @@ -895,8 +849,7 @@ SELECT 1, SELECT 1 FROM raw_contracts a WHERE a.source LIKE 'eop:%' AND a.contract_number = c.contract_number)) ) c - WHERE c.bidder_key IS NOT NULL - AND CASE c.value_flag + WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL @@ -906,6 +859,8 @@ SELECT 1, (SELECT COUNT(*) FROM contracts), datetime('now'); +DROP TABLE contractor_identity; + -- Summary (last result set printed by `wrangler d1 execute`) SELECT (SELECT COUNT(*) FROM authorities) AS authorities, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d16..ce27ead1c 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -158,7 +158,8 @@ SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6 FROM authority_totals at; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'contract', c.id, COALESCE(NULLIF(c.contract_subject, ''), t.title), COALESCE(t.source_id, ''), diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index a145895c5..4c383499a 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -45,61 +45,97 @@ UPDATE authorities SET type_group = CASE END WHERE type_group IS NULL; --- ── 2) Bidders referenced by OCDS staging (new ones only) — same identity rule as normalize step 4 ── +-- ── 2) Winning-contractor identity — derive the checksum and three-rung key once for this slice. ── +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +SELECT + eik_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) + ELSE 'unknown:анонимен' + END +FROM ( + SELECT + eik_raw, + name_raw, + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') + ) +); + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT bidder_key, - MIN(contractor_name), + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' ELSE MIN(name_raw) END, MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END -FROM ( - SELECT contractor_name, eik_clean, eik_valid, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, - CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' THEN 1 ELSE 0 - END AS grp - FROM ( - SELECT contractor_name, eik_clean, - -- Bulgarian ЕИК/Булстат checksum — identical to normalize-raw.sql step 4 (#195), so the - -- incremental path's bidder identity stays in parity with the full rebuild. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) -) -WHERE bidder_key IS NOT NULL + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity GROUP BY bidder_key ON CONFLICT(id) DO UPDATE SET name = CASE @@ -702,42 +738,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -767,6 +770,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'ocds:%' @@ -774,8 +779,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) -- EOP wins: skip OCDS rows when the transient window has an EOP row for the same document. @@ -977,42 +981,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - ( - SELECT - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END - FROM ( - SELECT - eik_clean, - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) AS eik_clean - ) - ) - ) AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -1042,6 +1013,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'eop:%' @@ -1061,8 +1034,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) AND NOT EXISTS ( @@ -1318,50 +1290,9 @@ INSERT OR IGNORE INTO refresh_touched_bidders (bidder_id) SELECT b.id FROM bidders b WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NULL) - OR b.id IN ( - SELECT bidder_key - FROM ( - SELECT - contractor_name, - CASE - WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key - FROM ( - SELECT - contractor_name, - eik_clean, - -- Same Bulgarian ЕИК checksum as the bidder INSERT above and normalize-raw.sql (#195), - -- so the touched-bidder key matches the actual bidders.id keying. - CASE - WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 - WHEN ( - CASE - WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 - WHEN LENGTH(eik_clean) = 9 THEN 1 - WHEN ( - CASE - WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 - ELSE 0 - END - ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 - ELSE 0 - END AS eik_valid - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) - ) - WHERE bidder_key IS NOT NULL - ); + OR b.id IN (SELECT bidder_key FROM contractor_identity); + +DROP TABLE contractor_identity; -- 6) Refresh rollups + FTS. Only the D1-hot rollups are scoped to touched rows; cheaper rollups stay -- full-recomputed in isolated batches so convergence stays simple. @@ -1407,7 +1338,8 @@ GROUP BY t.authority_id, c.bidder_id; DELETE FROM search_index WHERE kind = 'company'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; DELETE FROM search_index WHERE kind = 'authority'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6), ''), COALESCE(at.settlement, ''), at.spent_eur From efb1091f187eaeca72829dbd3902b88ab2c2f64d Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Sun, 19 Jul 2026 12:36:27 +0000 Subject: [PATCH 5/9] fix(etl): fold contractor name-key over case, quotes and dash variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name-keyed bidders (no valid ЕИК) split the same company across surface variants because SQLite UPPER() is ASCII-only and quote/dash encodings vary. Normalise the name in the `contractor_identity` scratch table so Cyrillic case, quote styles and dash encodings collapse to one key — while dash vs space stays distinct, so genuinely different names are not over-merged. - staged `name_norm` UPDATEs (whitespace + quotes + dashes + Cyrillic case), each a shallow expression: a single ~60-deep nested REPLACE overflows the sqlite3 CLI parser (v3.40.1) on the import path, so the fold is split into stages of <=15 REPLACE calls - the fold is byte-identical in normalize-raw and refresh-slice - tests assert quote/case/space variants merge to one key, en-dash == hyphen, and hyphen != space (over-fold guard); SUM(amount_eur) still reconciles --- .../db/src/contractor-identity-sql.test.ts | 32 +++++++++++++++++-- scripts/normalize-raw.sql | 32 +++++++++++++++++-- scripts/refresh-slice.sql | 32 +++++++++++++++++-- 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts index 1bcfbb16b..13aa10ea2 100644 --- a/packages/db/src/contractor-identity-sql.test.ts +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -29,7 +29,28 @@ VALUES '103267195', ''), ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-NULL', 'C-NULL', '2026-07-01', '2026-07-02', '123456786', 'Тестов възложител', 'Липсваща самоличност', 400, 400, 'EUR', - NULL, NULL); + NULL, NULL), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-1', 'C-FOLD-1', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 1', 10, 10, 'EUR', + '', '„Строй Инвест" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-2', 'C-FOLD-2', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 2', 10, 10, 'EUR', + '', '"СТРОЙ ИНВЕСТ" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-3', 'C-FOLD-3', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и интервали', 10, 10, 'EUR', + '', '«строй инвест» ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-EN', 'C-DASH-EN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 1', 10, 10, 'EUR', + '', '„Марица–Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-ASCII', 'C-DASH-ASCII', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 2', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-HYPHEN', 'C-GUARD-HYPHEN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази тирето', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-SPACE', 'C-GUARD-SPACE', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази интервала', 10, 10, 'EUR', + '', '„Марица Изток"'); `; function build(path: 'normalize' | 'refresh'): DatabaseSync { @@ -55,7 +76,14 @@ describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path .all() as { contract_number: string; bidder_id: string }[]; expect(contracts).toEqual([ + { contract_number: 'C-DASH-ASCII', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-DASH-EN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, { contract_number: 'C-EMPTY', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-FOLD-1', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-2', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-3', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-GUARD-HYPHEN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-GUARD-SPACE', bidder_id: 'name:МАРИЦА ИЗТОК' }, { contract_number: 'C-NULL', bidder_id: 'unknown:анонимен' }, { contract_number: 'C-TYPO', bidder_id: 'name:ТИПО ООД' }, { contract_number: 'C-VALID', bidder_id: 'eik:103267194' }, @@ -87,7 +115,7 @@ describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path (SELECT SUM(won_eur) FROM company_totals) AS company_total`, ) .get() as { raw_total: number; contract_total: number; company_total: number }; - expect(sums).toEqual({ raw_total: 1000, contract_total: 1000, company_total: 1000 }); + expect(sums).toEqual({ raw_total: 1070, contract_total: 1070, company_total: 1070 }); expect( db diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index c72a5f2a9..a6f9136ac 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -235,22 +235,23 @@ DROP TABLE IF EXISTS contractor_identity; CREATE TABLE contractor_identity ( eik_raw TEXT, name_raw TEXT, + name_norm TEXT, eik_clean TEXT, eik_valid INTEGER NOT NULL, bidder_key TEXT NOT NULL, PRIMARY KEY (eik_raw, name_raw) ); -INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) SELECT eik_raw, name_raw, + name_raw, eik_clean, eik_valid, CASE WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) - ELSE 'unknown:анонимен' + ELSE 'pending' END FROM ( SELECT @@ -295,6 +296,31 @@ FROM ( ) ); +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + -- 4a) Bidders — valid ЕИК first, then normalised name, then one labelled unknown bucket. The -- bucket keeps identity-poor contracts attached without polluting bulstat/eik_normalized. INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 4c383499a..c1e074d1d 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -50,22 +50,23 @@ DROP TABLE IF EXISTS contractor_identity; CREATE TABLE contractor_identity ( eik_raw TEXT, name_raw TEXT, + name_norm TEXT, eik_clean TEXT, eik_valid INTEGER NOT NULL, bidder_key TEXT NOT NULL, PRIMARY KEY (eik_raw, name_raw) ); -INSERT INTO contractor_identity (eik_raw, name_raw, eik_clean, eik_valid, bidder_key) +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) SELECT eik_raw, name_raw, + name_raw, eik_clean, eik_valid, CASE WHEN eik_valid = 1 THEN 'eik:' || eik_clean - WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(name_raw, ' ', ' '), ' ', ' '))) - ELSE 'unknown:анонимен' + ELSE 'pending' END FROM ( SELECT @@ -110,6 +111,31 @@ FROM ( ) ); +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT bidder_key, From 48e3302e1a46f683938fc6daff44557adbf9ac92 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Wed, 22 Jul 2026 14:55:48 +0000 Subject: [PATCH 6/9] fix(db): exclude the unknown-bidder bucket from authority top-suppliers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getAuthority's top-suppliers query lacked the AND b.kind <> 'unknown' guard the other three surfaces (home, companies, search_index) have, so the synthetic 'Неизвестен изпълнител' bucket could rank in an authority's public top-7. Add the same guard. --- packages/db/src/queries/details.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5e..960f9fbd0 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -248,7 +248,7 @@ export async function getAuthority( .prepare( `SELECT c.bidder_id, b.name, b.kind, SUM(c.amount_eur) AS won, COUNT(*) AS n FROM contracts c JOIN tenders t ON t.id = c.tender_id JOIN bidders b ON b.id = c.bidder_id - WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL + WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL AND b.kind <> 'unknown' GROUP BY c.bidder_id ORDER BY won DESC LIMIT 7`, ) .bind(authorityId) From d9e291eaef111d232b0831f9d6a7a4717478e1fc Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 13:04:30 +0000 Subject: [PATCH 7/9] fix(etl): exclude composite joint EIKs from authority minting A raw authority_eik can carry a joint-procurement composite ('EIK1; EIK2'). The rewritten canonical name/kind sources and the main authority insert consumed it verbatim, minting orphan 'auth:EIK1; EIK2' authorities referenced by no tender, contract, rollup or search row - a full 2020-2026 rebuild measured 404 of them. Guard every minting source with NOT LIKE '%;%'; joint tenders are attributed through their individual members. --- scripts/normalize-raw.sql | 11 +++++++---- scripts/refresh-slice.sql | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index 0b4f1018d..a4d49c1f6 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -66,9 +66,9 @@ FROM ( FROM ( SELECT authority_eik, authority_name, COUNT(*) AS cnt FROM ( - SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name FROM raw_contracts WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION ALL - SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT authority_eik, authority_name FROM raw_tenders WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) WHERE authority_name IS NOT NULL AND TRIM(authority_name) <> '' GROUP BY authority_eik, authority_name @@ -116,9 +116,12 @@ SELECT s.authority_eik, act.canonical_type FROM ( - SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL + -- Composite joint-procurement EIKs ('EIK1; EIK2') must not mint standalone authorities — + -- they are attributed via their individual members; an unguarded source mints orphan + -- 'auth:EIK1; EIK2' rows referenced by nothing (verified on a full 2020-2026 rebuild). + SELECT authority_eik FROM raw_contracts WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION - SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL + SELECT authority_eik FROM raw_tenders WHERE authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) s LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index a79a2ea7f..48d5fbbdf 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -46,7 +46,7 @@ FROM ( SELECT source, authority_eik, authority_name FROM raw_tenders ) WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') - AND authority_eik IS NOT NULL + AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' AND authority_name IS NOT NULL AND TRIM(authority_name) <> '' GROUP BY authority_eik, authority_name ) @@ -93,11 +93,12 @@ SELECT s.authority_eik, act.canonical_type FROM ( + -- Composite joint EIKs must not mint standalone authorities (see normalize-raw.sql). SELECT authority_eik FROM raw_contracts - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' UNION SELECT authority_eik FROM raw_tenders - WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL + WHERE (source LIKE 'eop:%' OR source LIKE 'ocds:%') AND authority_eik IS NOT NULL AND authority_eik NOT LIKE '%;%' ) s LEFT JOIN authority_canonical_name acn ON acn.authority_eik = s.authority_eik LEFT JOIN authority_canonical_type act ON act.authority_eik = s.authority_eik; From c06761e5bbbeeae1650e7d90eab2e500cb603586 Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 19:59:43 +0000 Subject: [PATCH 8/9] chore: drop accidentally committed node_modules symlinks The .gitignore pattern node_modules/ matches directories only; a dev-environment symlink slipped past it into the merge commit and broke CI's pnpm install (ENOTDIR). Remove from tracking. --- node_modules | 1 - packages/db/node_modules | 1 - 2 files changed, 2 deletions(-) delete mode 120000 node_modules delete mode 120000 packages/db/node_modules diff --git a/node_modules b/node_modules deleted file mode 120000 index 6df2b27f4..000000000 --- a/node_modules +++ /dev/null @@ -1 +0,0 @@ -/workspaces/sigma/node_modules \ No newline at end of file diff --git a/packages/db/node_modules b/packages/db/node_modules deleted file mode 120000 index 5f54326b2..000000000 --- a/packages/db/node_modules +++ /dev/null @@ -1 +0,0 @@ -/workspaces/sigma/packages/db/node_modules \ No newline at end of file From 45f9040c11c7ff2c704f746565e8207a6af29b6f Mon Sep 17 00:00:00 2001 From: Todor Kolev Date: Thu, 23 Jul 2026 20:02:47 +0000 Subject: [PATCH 9/9] style: prettier-format companies.test.ts --- packages/db/src/queries/companies.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index b0e9b9f0e..f6e038e97 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -120,9 +120,9 @@ describe('streamCompaniesCsv', () => { expect(sql.filter((query) => query.includes('company_totals'))).not.toHaveLength(0); expect( - sql.filter((query) => query.includes('company_totals')).every((query) => - query.includes("kind <> 'unknown'"), - ), + sql + .filter((query) => query.includes('company_totals')) + .every((query) => query.includes("kind <> 'unknown'")), ).toBe(true); }); });