diff --git a/packages/db/src/competition-sql.test.ts b/packages/db/src/competition-sql.test.ts index baa0fdb48..20d05aad2 100644 --- a/packages/db/src/competition-sql.test.ts +++ b/packages/db/src/competition-sql.test.ts @@ -31,10 +31,10 @@ function readScript(dbPath: string, path: string): void { const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:A', 'Институция А', '100000001', 'община'), - ('auth:B', 'Институция Б', '100000002', 'община'); + ('auth:B', 'Институция Б', '100000019', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:X', 'Фирма Х', '200000001', '200000001', 1, 'company'), - ('eik:Y', 'Фирма У', '200000002', '200000002', 1, 'company'); + ('eik:X', 'Фирма Х', '200000002', '200000002', 1, 'company'), + ('eik:Y', 'Фирма У', '200000015', '200000015', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:A', '45', 'открита процедура', 'awarded'), ('t:B', 'UNP-B', 'Поръчка Б', 'auth:B', '45', 'открита процедура', 'awarded'); diff --git a/packages/db/src/contractor-identity-sql.test.ts b/packages/db/src/contractor-identity-sql.test.ts new file mode 100644 index 000000000..13aa10ea2 --- /dev/null +++ b/packages/db/src/contractor-identity-sql.test.ts @@ -0,0 +1,132 @@ +/// +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schema = readFileSync(resolve(root, 'packages/db/migrations/0000_init.sql'), 'utf8'); +const staging = readFileSync(resolve(root, 'scripts/work-staging-schema.sql'), 'utf8'); +const normalize = readFileSync(resolve(root, 'scripts/normalize-raw.sql'), 'utf8'); +const precompute = readFileSync(resolve(root, 'scripts/precompute.sql'), 'utf8'); +const refresh = readFileSync(resolve(root, 'scripts/refresh-slice.sql'), 'utf8'); + +const seed = ` +INSERT INTO raw_contracts + (source, fetched_at, unp, contract_number, contract_date, published_at, + authority_eik, authority_name, procurement_subject, estimated_value, + signing_value, currency, contractor_eik, contractor_name) +VALUES + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-VALID', 'C-VALID', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Валиден ЕИК', 100, 100, 'EUR', + '103267194', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-TYPO', 'C-TYPO', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Грешна контролна цифра', 200, 200, 'EUR', + '103267195', 'ТИПО ООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-EMPTY', 'C-EMPTY', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Празно име', 300, 300, 'EUR', + '103267195', ''), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-NULL', 'C-NULL', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Липсваща самоличност', 400, 400, 'EUR', + NULL, NULL), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-1', 'C-FOLD-1', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 1', 10, 10, 'EUR', + '', '„Строй Инвест" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-2', 'C-FOLD-2', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и регистър 2', 10, 10, 'EUR', + '', '"СТРОЙ ИНВЕСТ" ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-FOLD-3', 'C-FOLD-3', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Кавички и интервали', 10, 10, 'EUR', + '', '«строй инвест» ЕООД'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-EN', 'C-DASH-EN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 1', 10, 10, 'EUR', + '', '„Марица–Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-DASH-ASCII', 'C-DASH-ASCII', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Тире 2', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-HYPHEN', 'C-GUARD-HYPHEN', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази тирето', 10, 10, 'EUR', + '', '„Марица-Изток"'), + ('eop:contracts:test', '2026-07-18T00:00:00Z', 'UNP-GUARD-SPACE', 'C-GUARD-SPACE', '2026-07-01', + '2026-07-02', '123456786', 'Тестов възложител', 'Пази интервала', 10, 10, 'EUR', + '', '„Марица Изток"'); +`; + +function build(path: 'normalize' | 'refresh'): DatabaseSync { + const db = new DatabaseSync(':memory:'); + db.exec(schema); + db.exec(staging); + db.exec(seed); + if (path === 'normalize') { + db.exec(normalize); + db.exec(precompute); + } else { + db.exec(refresh); + } + return db; +} + +describe.each(['normalize', 'refresh'] as const)('%s contractor identity', (path) => { + it('preserves every eligible contract across valid, typo, empty, and NULL identities', () => { + const db = build(path); + try { + const contracts = db + .prepare('SELECT contract_number, bidder_id FROM contracts ORDER BY contract_number') + .all() as { contract_number: string; bidder_id: string }[]; + + expect(contracts).toEqual([ + { contract_number: 'C-DASH-ASCII', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-DASH-EN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-EMPTY', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-FOLD-1', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-2', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-FOLD-3', bidder_id: 'name:СТРОЙ ИНВЕСТ ЕООД' }, + { contract_number: 'C-GUARD-HYPHEN', bidder_id: 'name:МАРИЦА-ИЗТОК' }, + { contract_number: 'C-GUARD-SPACE', bidder_id: 'name:МАРИЦА ИЗТОК' }, + { contract_number: 'C-NULL', bidder_id: 'unknown:анонимен' }, + { contract_number: 'C-TYPO', bidder_id: 'name:ТИПО ООД' }, + { contract_number: 'C-VALID', bidder_id: 'eik:103267194' }, + ]); + + expect( + db + .prepare( + `SELECT id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind + FROM bidders WHERE id = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ + id: 'unknown:анонимен', + name: 'Неизвестен изпълнител', + bulstat: null, + eik_normalized: null, + eik_valid: 0, + is_consortium: 0, + kind: 'unknown', + }); + + const sums = db + .prepare( + `SELECT + (SELECT SUM(signing_value) FROM raw_contracts + WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') AS raw_total, + (SELECT SUM(amount_eur) FROM contracts) AS contract_total, + (SELECT SUM(won_eur) FROM company_totals) AS company_total`, + ) + .get() as { raw_total: number; contract_total: number; company_total: number }; + expect(sums).toEqual({ raw_total: 1070, contract_total: 1070, company_total: 1070 }); + + expect( + db + .prepare( + `SELECT COUNT(*) AS n FROM search_index + WHERE kind = 'company' AND ref = 'unknown:анонимен'`, + ) + .get(), + ).toEqual({ n: 0 }); + } finally { + db.close(); + } + }); +}); diff --git a/packages/db/src/etl-entity-canonicalization-sql.test.ts b/packages/db/src/etl-entity-canonicalization-sql.test.ts index ca85461b5..66bf3a17c 100644 --- a/packages/db/src/etl-entity-canonicalization-sql.test.ts +++ b/packages/db/src/etl-entity-canonicalization-sql.test.ts @@ -134,7 +134,7 @@ describe('ETL entity canonicalization through real SQL scripts', () => { }); it('uses the bidder-name mode for EIK- and name-keyed bidders in both ETL paths', () => { - const bidderEik = '400000001'; + const bidderEik = '400000004'; for (const [label, scriptPath] of etlPaths) { withEtlDb(label, (dbPath) => { diff --git a/packages/db/src/queries/companies.test.ts b/packages/db/src/queries/companies.test.ts index a34c1ad07..f6e038e97 100644 --- a/packages/db/src/queries/companies.test.ts +++ b/packages/db/src/queries/companies.test.ts @@ -4,11 +4,11 @@ import type { CompanyTotalsRow } from './rows'; const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ { - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', name: 'Филтрирана фирма', kind: 'company', ownership_kind: null, - eik: '111111111', + eik: '111111113', eik_valid: 1, settlement: 'София', won_eur: 1000, @@ -25,11 +25,11 @@ const filteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ const unfilteredRows: (CompanyTotalsRow & { sort_value: number })[] = [ ...filteredRows, { - bidder_id: 'eik:999999999', + bidder_id: 'eik:999999995', name: 'Нефилтрирана фирма', kind: 'company', ownership_kind: null, - eik: '999999999', + eik: '999999995', eik_valid: 1, settlement: 'Пловдив', won_eur: 900, @@ -93,7 +93,7 @@ describe('streamCompaniesCsv', () => { .map((line) => line.split(',')[0]); expect(csvEiks).toEqual(list.items.map((item) => item.eik)); - expect(csvEiks).toEqual(['111111111']); + expect(csvEiks).toEqual(['111111113']); }); it('exports fewer rows for a sector filter than for the unfiltered corpus', async () => { @@ -105,6 +105,26 @@ describe('streamCompaniesCsv', () => { expect(countRows(filtered)).toBeLessThan(countRows(unfiltered)); expect(countRows(filtered)).toBe(1); }); + + it('excludes the unknown identity bucket from the list and CSV', async () => { + const db = fakeDb(); + const sql: string[] = []; + const real = db.prepare.bind(db); + db.prepare = ((query: string) => { + sql.push(query); + return real(query); + }) as typeof db.prepare; + + await listCompanies(db, {}); + await streamCompaniesCsv(db, {}).text(); + + expect(sql.filter((query) => query.includes('company_totals'))).not.toHaveLength(0); + expect( + sql + .filter((query) => query.includes('company_totals')) + .every((query) => query.includes("kind <> 'unknown'")), + ).toBe(true); + }); }); describe('prototype-key params (untrusted query values)', () => { diff --git a/packages/db/src/queries/companies.ts b/packages/db/src/queries/companies.ts index 70a1fa401..8f2665238 100644 --- a/packages/db/src/queries/companies.ts +++ b/packages/db/src/queries/companies.ts @@ -109,7 +109,7 @@ function source(p: CompanyListParams): { from: string; params: unknown[] } { * Keep consumed filter keys in sync with COMPANY_FILTER_KEYS and companyFilterSignature(). */ function entityWhere(p: CompanyListParams): { sql: string; params: unknown[] } { - const where: string[] = []; + const where: string[] = [`kind <> 'unknown'`]; const params: unknown[] = []; if (p.kinds?.length && p.kinds.length < 2) { where.push(`kind = ?`); diff --git a/packages/db/src/queries/contracts-filter-sql.test.ts b/packages/db/src/queries/contracts-filter-sql.test.ts index 2c4d69746..98314206e 100644 --- a/packages/db/src/queries/contracts-filter-sql.test.ts +++ b/packages/db/src/queries/contracts-filter-sql.test.ts @@ -26,14 +26,14 @@ const FIXTURE = ` INSERT INTO authorities (id, name, bulstat, type_group) VALUES ('auth:100000001', 'Институция А', '100000001', 'община'); INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES - ('eik:200000001', 'Фирма Х', '200000001', '200000001', 1, 'company'); + ('eik:200000002', 'Фирма Х', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:100000001', '45000000', 'открита процедура', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, bids_received, value_flag, amount_eur) VALUES - ('c:NULL', 't:A', 'eik:200000001', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), - ('c:ZERO', 't:A', 'eik:200000001', 200, 'EUR', '2024-01-02', 0, 'ok', 200), - ('c:ONE', 't:A', 'eik:200000001', 300, 'EUR', '2024-01-03', 1, 'ok', 300), - ('c:TWO', 't:A', 'eik:200000001', 400, 'EUR', '2024-01-04', 2, 'ok', 400); + ('c:NULL', 't:A', 'eik:200000002', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), + ('c:ZERO', 't:A', 'eik:200000002', 200, 'EUR', '2024-01-02', 0, 'ok', 200), + ('c:ONE', 't:A', 'eik:200000002', 300, 'EUR', '2024-01-03', 1, 'ok', 300), + ('c:TWO', 't:A', 'eik:200000002', 400, 'EUR', '2024-01-04', 2, 'ok', 400); `; /** Minimal D1Database facade over node:sqlite — enough for the query layer's prepare/bind/all/first. */ diff --git a/packages/db/src/queries/contracts.test.ts b/packages/db/src/queries/contracts.test.ts index 504754b24..d23391116 100644 --- a/packages/db/src/queries/contracts.test.ts +++ b/packages/db/src/queries/contracts.test.ts @@ -21,9 +21,9 @@ const contractRow = { unp: 'UNP-1', cpv_code: '45000000', eu_funded: 0, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, procedure_type: 'Открита процедура', diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index 04e3cd009..ffe35cffe 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -41,14 +41,14 @@ const baseContractRow = { tender_fx_rate: null as number | null, start_date: null, end_date: null, - authority_id: 'auth:123456789', + authority_id: 'auth:123456786', authority_name: 'Authority', authority_type_group: 'ministry', authority_settlement: 'Sofia', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', bidder_name: 'Bidder', bidder_kind: 'company' as const, - bidder_eik: '111111111', + bidder_eik: '111111113', bidder_settlement: 'Sofia', }; @@ -102,7 +102,7 @@ describe('getContract', () => { estimated_fx_rate: null, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', @@ -147,7 +147,7 @@ describe('getContract', () => { estimated_fx_rate: 0.9, bidder_name: 'Bidder', bidder_kind: 'company', - bidder_id: 'eik:111111111', + bidder_id: 'eik:111111113', }, { lot_id: 'lot:UNP-1:2', diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 71f8347e3..753db1573 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -248,7 +248,7 @@ export async function getAuthority( .prepare( `SELECT c.bidder_id, b.name, b.kind, SUM(c.amount_eur) AS won, COUNT(*) AS n FROM contracts c JOIN tenders t ON t.id = c.tender_id JOIN bidders b ON b.id = c.bidder_id - WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL + WHERE t.authority_id = ? AND c.amount_eur IS NOT NULL AND b.kind <> 'unknown' GROUP BY c.bidder_id ORDER BY won DESC LIMIT 7`, ) .bind(authorityId) diff --git a/packages/db/src/queries/home.test.ts b/packages/db/src/queries/home.test.ts index 0302eb457..257c8be50 100644 --- a/packages/db/src/queries/home.test.ts +++ b/packages/db/src/queries/home.test.ts @@ -61,9 +61,10 @@ const totalsRow = { refreshed_at: '2024-06-02T10:00:00Z', }; -function fakeDb(totals: typeof totalsRow | null): D1Database { +function fakeDb(totals: typeof totalsRow | null, capture?: string[]): D1Database { return { prepare(sql: string) { + capture?.push(sql); return { bind() { return this; @@ -121,6 +122,15 @@ describe('getHomeData', () => { expect(data.topMunicipalities).toHaveLength(1); }); + it('excludes the unknown identity bucket from top companies', async () => { + const sql: string[] = []; + await getHomeData(fakeDb(totalsRow, sql)); + + expect(sql.find((query) => query.includes('FROM company_totals'))).toContain( + "WHERE kind <> 'unknown'", + ); + }); + it('includes single-offer contract lists', async () => { const data = await getHomeData(fakeDb(totalsRow)); diff --git a/packages/db/src/queries/home.ts b/packages/db/src/queries/home.ts index 7b9a9ee46..bcfbad632 100644 --- a/packages/db/src/queries/home.ts +++ b/packages/db/src/queries/home.ts @@ -53,7 +53,9 @@ export async function getHomeData(db: D1Database): Promise { const [companies, ministries, municipalities, recentSingleOffer, topSingleOffer, singleOfferRow] = await Promise.all([ db - .prepare(`SELECT * FROM company_totals ORDER BY won_eur DESC, bidder_id LIMIT 10`) + .prepare( + `SELECT * FROM company_totals WHERE kind <> 'unknown' ORDER BY won_eur DESC, bidder_id LIMIT 10`, + ) .all(), db .prepare( diff --git a/packages/db/src/queries/identity.test.ts b/packages/db/src/queries/identity.test.ts index 0cdc42726..7ca09ed7b 100644 --- a/packages/db/src/queries/identity.test.ts +++ b/packages/db/src/queries/identity.test.ts @@ -23,7 +23,11 @@ describe('company slug', () => { expect(bidderIdFromSlug(slug)).toBe(id); }); it('round-trips a 13-digit ЕИК', () => { - expect(bidderIdFromSlug(companySlug('eik:1234567890123'))).toBe('eik:1234567890123'); + expect(bidderIdFromSlug(companySlug('eik:8316417910124'))).toBe('eik:8316417910124'); + }); + it('resolves the unknown bidder linked from contract pages', () => { + expect(companySlug('unknown:анонимен')).toBe('unknown:анонимен'); + expect(bidderIdFromSlug('unknown:анонимен')).toBe('unknown:анонимен'); }); }); diff --git a/packages/db/src/queries/identity.ts b/packages/db/src/queries/identity.ts index a12067b33..2780bfc53 100644 --- a/packages/db/src/queries/identity.ts +++ b/packages/db/src/queries/identity.ts @@ -21,7 +21,8 @@ function b64urlDecode(s: string): string { const EIK_RE = /^\d{9}(\d{4})?$/; -/** bidder id → `/companies/:slug` segment. Valid ЕИК → the digits; name-keyed → `n` + base64url(name). */ +/** bidder id → `/companies/:slug` segment. Valid ЕИК → digits; name-keyed → `n` + base64url(name); + * the labelled unknown bucket stays literal so contract-page links remain resolvable. */ export function companySlug(bidderId: string): string { if (bidderId.startsWith('eik:')) return bidderId.slice(4); if (bidderId.startsWith('name:')) return 'n' + b64urlEncode(bidderId.slice(5)); @@ -31,6 +32,7 @@ export function companySlug(bidderId: string): string { /** `/companies/:slug` segment → bidder id, or null if it cannot be decoded. */ export function bidderIdFromSlug(slug: string): string | null { if (EIK_RE.test(slug)) return 'eik:' + slug; + if (slug === 'unknown:анонимен') return slug; if (slug.startsWith('n')) { try { return 'name:' + b64urlDecode(slug.slice(1)); diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index c654d8b9b..8470d55df 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -64,11 +64,11 @@ INSERT INTO raw_tenders VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', 'Construction', 'works', 2000, 'BGN', 'basis', - 'lowest', 'Authority CE', '123456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority CE', '123456786', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -82,12 +82,12 @@ INSERT INTO raw_contracts VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-CE-1', '2026-06-01', 'UNP-CE-1', 'TENDER-CE-1', 'open', 'Base tender', '45000000', - 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 2000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', '1', 'CONTRACT-CE-1', '2026-06-02', 1000, 'BGN', - 'Base contract', 0, '987654321', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Base contract', 0, '987654308', 'Bidder CE', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', 0, 'DOC-CO-1', '2026-06-01', 'OCDS-CO-1', 'TENDER-CO-1', 'open', 'OCDS tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority CE', '123456786', 'public', 'activity', 'notice', NULL, 'CONTRACT-CO-1', '2026-06-02', 1000, 'BGN', 'OCDS contract', 0, '987654322', 'Bidder CO', 'BG', 'small', 0, 3, 1, 0, 0, 30); @@ -98,10 +98,10 @@ INSERT INTO raw_amendments currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase'), ('ocds:2026-06-01', 2026, 'ocds', '2026-06-07T00:00:00Z', '1', 'AMD-CO-1', - 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456789', 'Authority CE', + 'CONTRACT-CO-1', '2026-06-02', '2026-06-03', 'OCDS-CO-1', '123456786', 'Authority CE', 'OCDS tender', 'works', 1000, 1300, 300, 'BGN', 'OCDS increase'); `, ); @@ -117,7 +117,7 @@ function seedRepeatedAnnexOnly(dbPath: string): void { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '1', 'AMD-CE-1', - 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456789', 'Authority CE', + 'CONTRACT-CE-1', '2026-06-02', '2026-06-03', 'UNP-CE-1', '123456786', 'Authority CE', 'Base tender', 'works', 1000, 1200, 200, 'BGN', 'Increase');`, ); } @@ -133,7 +133,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', 'Construction', 'works', 5000, 'BGN', 'basis', - 'lowest', 'Authority Shared', '223456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Shared', '223456787', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -147,7 +147,7 @@ function seedEopOnlySharedNumber(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-SHARED', '2026-06-01', 'UNP-SHARED', 'TENDER-SHARED', 'open', 'Shared tender', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -167,7 +167,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { VALUES ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-SHARED-O', '2026-06-02', 'OCDS-SHARED', 'TENDER-SHARED-O', 'open', 'Shared tender ocds', '45000000', - 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456789', + 'Construction', 'works', 5000, 'BGN', 'basis', 'lowest', 'Authority Shared', '223456787', 'public', 'activity', 'notice', NULL, 'CONTRACT-SHARED', '2026-06-02', 1000, 'BGN', 'Shared contract ocds', 0, '887654321', 'Bidder Shared', 'BG', 'small', 0, 3, 1, 0, 0, 30);`, ); @@ -189,11 +189,11 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'), ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority ID', '523456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority ID', '523456782', 'public', 'activity', '2026-06-10', 'notice', '1', 'Lot 1', 1, 0, '2026-06-01'); INSERT INTO raw_contracts @@ -207,29 +207,29 @@ function seedContractIdFixture(dbPath: string): void { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-OLD', '2026-06-01', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'Old duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'Old duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NEW', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', '1', 'CONTRACT-ID', '2026-06-02', 1000, 'BGN', - 'New duplicate', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), + 'New duplicate', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 3, 1, 0, 0, 30), ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-NULL', '2026-06-02', 'UNP-ID', 'TENDER-ID', 'open', 'ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, NULL, '2026-06-03', 500, 'BGN', 'Null number', 0, '577777778', 'Bidder Null', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS', '2026-06-02', 'OCDS-ID', 'TENDER-OCDS', 'open', 'OCDS ID tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', 'LOT-0001', 'CONTRACT-OCDS', '2026-06-04', 700, 'BGN', - 'OCDS ID', 0, '577777779', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), + 'OCDS ID', 0, '577777792', 'Bidder OCDS', 'BG', 'small', 0, 2, 1, 0, 0, 30), ('ocds:2026-06-02', 2026, 'ocds', '2026-06-08T00:00:00Z', 0, 'DOC-OCDS-SKIP', '2026-06-02', 'OCDS-ID-SKIP', 'TENDER-OCDS-SKIP', 'open', 'OCDS skip tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority ID', '523456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-ID', '2026-06-04', 700, 'BGN', - 'OCDS skip', 0, '577777777', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, + 'OCDS skip', 0, '577777760', 'Bidder ID', 'BG', 'small', 0, 2, 1, 0, 0, 30);`, ); } @@ -259,7 +259,7 @@ function seedSyntheticWindow( '2026-06-02', ${sqlValue(values.unp)}, ${sqlValue(`TENDER-${values.unp}`)}, 'open', ${sqlValue(values.subject)}, ${sqlValue(values.cpv)}, 'Construction', 'works', ${sqlValue(values.estimated)}, ${sqlValue(values.currency)}, - 'basis', 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', 'notice', + 'basis', 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${values.source}`)}, '2026-06-02', 100, ${sqlValue(values.currency)}, 'Synthetic contract', 0, '667777777', 'Bidder Synthetic', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -269,7 +269,7 @@ function seedSyntheticAuthority(dbPath: string): void { sqlite( dbPath, `INSERT OR IGNORE INTO authorities (id, name, bulstat, type, type_group) - VALUES ('auth:623456789', 'Authority Synthetic', '623456789', 'public', 'other');`, + VALUES ('auth:623456780', 'Authority Synthetic', '623456780', 'public', 'other');`, ); } @@ -281,7 +281,7 @@ function seedReplayParty(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-replay', 'party-replay', - '823456789', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', + '823456782', 'Replay party', 'Replay street 1', 'Replay city', 'BG411', 'replay@example.test', '+359000000001'), ('ocds:2026-06-01', '2026-06-08T00:00:00Z', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);`, @@ -302,9 +302,9 @@ function seedReplayEntity(dbPath: string): void { VALUES ('eop:contracts:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', 0, 'DOC-REPLAY', '2026-06-02', 'UNP-REPLAY', 'TENDER-REPLAY', 'open', 'Replay tender', '45000000', - 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456789', + 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Replay authority', '823456782', 'public', 'activity', 'notice', NULL, 'CONTRACT-REPLAY', '2026-06-02', 100, 'BGN', - 'Replay contract', 0, '823456789', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, + 'Replay contract', 0, '823456782', 'Replay bidder', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); } @@ -320,16 +320,16 @@ function seedCollidingPartySlot(dbPath: string): void { region_nuts, contact_email, contact_phone) VALUES ('ocds:2026-06-01', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '111111111', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), + '111111113', 'Company A', 'Addr A 1', 'City A', 'BG331', 'a@example.test', '+359000000011'), ('ocds:2026-06-05', '2026-06-08T00:00:00Z', 'ocds-shared', 'ORG-0003', - '222222222', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, + '222222226', 'Company B', 'Addr B 2', 'City B', 'BG421', 'b@example.test', '+359000000022');`, ); } function seedTwoCollidingBidders(dbPath: string): void { for (const [eik, name] of [ - ['111111111', 'Company A'], - ['222222222', 'Company B'], + ['111111113', 'Company A'], + ['222222226', 'Company B'], ] as const) { sqlite( dbPath, @@ -345,7 +345,7 @@ function seedTwoCollidingBidders(dbPath: string): void { (${sqlValue(`eop:contracts:2026-06-02:${eik}`)}, 2026, 'eop', '2026-06-08T00:00:00Z', 0, ${sqlValue(`DOC-${eik}`)}, '2026-06-02', ${sqlValue(`UNP-${eik}`)}, ${sqlValue(`TENDER-${eik}`)}, 'open', 'Slot tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', - 'Slot authority', '923456789', 'public', 'activity', 'notice', NULL, + 'Slot authority', '923456783', 'public', 'activity', 'notice', NULL, ${sqlValue(`CONTRACT-${eik}`)}, '2026-06-02', 100, 'BGN', 'Slot contract', 0, ${sqlValue(eik)}, ${sqlValue(name)}, 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -488,15 +488,15 @@ describe('refresh-slice EOP base derivation', () => { readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, - `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456789', 'Authority Flag', '323456789', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777777', 'Bidder Flag', '777777777', '777777777', 1, 'company'); + `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:323456788', 'Authority Flag', '323456788', 'public'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:777777771', 'Bidder Flag', '777777771', '777777771', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) - VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456789', 1000, 'BGN', 'open', 'awarded'); + VALUES ('t:UNP-FLAG', 'UNP-FLAG', 'Flag tender', 'auth:323456788', 1000, 'BGN', 'open', 'awarded'); INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, contract_number, signing_value, current_value, annex_count, value_flag, amount_eur, signing_value_eur) VALUES - ('c:e:flag', 't:UNP-FLAG', 'eik:777777777', 100, 'BGN', '2026-06-02', + ('c:e:flag', 't:UNP-FLAG', 'eik:777777771', 100, 'BGN', '2026-06-02', 'CONTRACT-FLAG', 100, 10000, 1, 'annex_suspect', NULL, 100 / 1.95583); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -505,7 +505,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-02', 2026, 'eop', '2026-06-08T00:00:00Z', '2', 'AMD-FLAG-2', - 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456789', 'Authority Flag', + 'CONTRACT-FLAG', '2026-06-02', '2026-06-03', 'UNP-FLAG', '323456788', 'Authority Flag', 'Flag tender', 'works', 10000, 120, -9880, 'BGN', 'Normalize');`, ); @@ -560,7 +560,7 @@ describe('refresh-slice EOP base derivation', () => { '2026-06-01', 'UNP-MISMATCH', 'TENDER-MISMATCH', 'open', 'Mismatch tender', '45000000', 'Construction', 'works', 1000, 'BGN', 'basis', 'lowest', 'Authority Mismatch', '423456789', 'public', 'activity', 'notice', NULL, 'CONTRACT-MISMATCH', '2026-06-02', 1000, 'BGN', - 'Mismatch contract', 0, '677777777', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); + 'Mismatch contract', 0, '677777779', 'Bidder Mismatch', 'BG', 'small', 0, 3, 1, 0, 0, 30); INSERT INTO raw_amendments (source, dataset_year, dataset_variant, fetched_at, seq_no, document_number, @@ -660,7 +660,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', 'Construction', 'works', 10000, 'BGN', 'basis', - 'lowest', 'Authority Ordinal', '723456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01'); INSERT INTO raw_amendments @@ -670,7 +670,7 @@ describe('refresh-slice EOP base derivation', () => { currency, description) VALUES ('eop:annexes:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', '1', 'AMD-ORD', - 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456789', 'Authority Ordinal', + 'CONTRACT-ORD', '2026-06-02', '2026-06-03', 'UNP-ORD', '723456781', 'Authority Ordinal', 'Ordinal tender', 'works', 100, 150, 50, 'BGN', 'Increase');`, ); } @@ -687,12 +687,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, 150, 1, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -709,12 +709,12 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-A', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal A', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30), ('eop:contracts:2026-06-01', 2026, 'eop', '2026-06-07T00:00:00Z', 0, 'DOC-B', '2026-06-01', 'UNP-ORD', 'TENDER-ORD', 'open', 'Ordinal tender', '45000000', - 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456789', + 'Construction', 'works', 10000, 'BGN', 'basis', 'lowest', 'Authority Ordinal', '723456781', 'public', 'activity', 'notice', NULL, 'CONTRACT-ORD', '2026-06-02', 100, NULL, 0, 'BGN', 'Ordinal B', 0, '777777771', 'Bidder Ordinal', 'BG', 'small', 0, 1, 1, 0, 0, 30);`, ); @@ -833,7 +833,7 @@ describe('refresh-slice EOP base derivation', () => { VALUES ('eop:tenders:2026-06-01', 2026, '2026-06-07T00:00:00Z', 'UNP-REAL', 'TENDER-REAL', NULL, 'Real header', '99000000', 'Header', 'works', 990, 'EUR', 'basis', - 'lowest', 'Authority Synthetic', '623456789', 'public', 'activity', '2026-06-10', 'notice', + 'lowest', 'Authority Synthetic', '623456780', 'public', 'activity', '2026-06-10', 'notice', NULL, NULL, 1, 0, '2026-06-01');`, ); seedSyntheticWindow(dbPath, { @@ -884,11 +884,17 @@ describe('refresh-slice EOP base derivation', () => { readScript(sliceDb, refreshSlicePath); const authoritySql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456789'"; + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM authorities WHERE id='auth:823456782'"; const bidderSql = - "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456789'"; - expect(sqliteJson(fullDb, authoritySql)).toEqual(sqliteJson(sliceDb, authoritySql)); - expect(sqliteJson(fullDb, bidderSql)).toEqual(sqliteJson(sliceDb, bidderSql)); + "SELECT id, nuts, settlement, address, contact_email, contact_phone FROM bidders WHERE id='eik:823456782'"; + const fullAuthorities = sqliteJson(fullDb, authoritySql); + const sliceAuthorities = sqliteJson(sliceDb, authoritySql); + const fullBidders = sqliteJson(fullDb, bidderSql); + const sliceBidders = sqliteJson(sliceDb, bidderSql); + expect(fullAuthorities).not.toHaveLength(0); + expect(sliceAuthorities).toEqual(fullAuthorities); + expect(fullBidders).not.toHaveLength(0); + expect(sliceBidders).toEqual(fullBidders); const partiesAfterWindow2 = sqliteJson(sliceDb, 'SELECT * FROM parties ORDER BY party_key'); const authorityAfterWindow2 = sqliteJson(sliceDb, authoritySql); @@ -920,19 +926,19 @@ describe('refresh-slice EOP base derivation', () => { const sliceDb = resolve(dir, 'slice.sqlite'); try { const partyByEik = - "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111111','222222222') ORDER BY eik"; + "SELECT eik, locality, street_address, contact_email FROM parties WHERE eik IN ('111111113','222222226') ORDER BY eik"; const bidderByEik = - "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111111','222222222') ORDER BY eik_normalized"; + "SELECT eik_normalized AS eik, settlement, address, contact_email FROM bidders WHERE eik_normalized IN ('111111113','222222226') ORDER BY eik_normalized"; // Each company must be enriched from its OWN party row, not the latest slot occupant's. const expectedParties = [ { - eik: '111111111', + eik: '111111113', locality: 'City A', street_address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', locality: 'City B', street_address: 'Addr B 2', contact_email: 'b@example.test', @@ -940,13 +946,13 @@ describe('refresh-slice EOP base derivation', () => { ]; const expectedBidders = [ { - eik: '111111111', + eik: '111111113', settlement: 'City A', address: 'Addr A 1', contact_email: 'a@example.test', }, { - eik: '222222222', + eik: '222222226', settlement: 'City B', address: 'Addr B 2', contact_email: 'b@example.test', @@ -980,7 +986,7 @@ function seedReattrContract( source: string, eik: string, name: string, - authorityEik = '923456789', + authorityEik = '923456783', authorityName = 'Reattr authority', ): void { sqlite( @@ -1042,18 +1048,18 @@ describe('refresh-slice integrity gate', () => { try { initWorkDb(dbPath); // window 1: contract attributed to bidder A - seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111111', 'Company A'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-02', '111111113', 'Company A'); readScript(dbPath, refreshSlicePath); - expect(wonEur('111111111')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBeGreaterThan(0); // window 2: same contract_number+unp re-imported with a different contractor → re-attribution resetRawStaging(dbPath); - seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222222', 'Company B'); + seedReattrContract(dbPath, 'eop:contracts:2026-06-05', '222222226', 'Company B'); readScript(dbPath, refreshSlicePath); // A's scoped rollup was rebuilt to empty (the value moved), B now carries it… - expect(wonEur('111111111')).toBe(0); - expect(wonEur('222222222')).toBeGreaterThan(0); + expect(wonEur('111111113')).toBe(0); + expect(wonEur('222222226')).toBeGreaterThan(0); // …so the gate reconciles on the slice-built DB (rollup check runs; staging self-skips here). const results = await assertIntegrity(run, { label: 'test-slice', exit: false }); diff --git a/packages/db/src/ship-domain.test.ts b/packages/db/src/ship-domain.test.ts index e1249ed7d..7101ec83c 100644 --- a/packages/db/src/ship-domain.test.ts +++ b/packages/db/src/ship-domain.test.ts @@ -49,9 +49,9 @@ describe('ship-domain', () => { workDb, `INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:1', 'Authority line 1 Authority line 2', '1', 'public'); - INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000007', 'Bidder', '200000007', '200000007', 1, 'company'); + INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000002', 'Bidder', '200000002', '200000002', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, currency, procedure_type, status) VALUES ('t:1', '1', 'Tender', 'auth:1', 'BGN', 'open', 'awarded'); - INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000007', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); + INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, contract_number, signing_value, value_flag, amount_eur) VALUES ('c:e:1', 't:1', 'eik:200000002', 10, 'BGN', 'C1', 10, 'ok', 10 / 1.95583); INSERT INTO amendments (id, natural_key, contract_number, unp, description, source) VALUES ('am:1:C1:A1', 'am:1:C1:A1', 'C1', '1', 'Description line 1 Description line 2', 'test'); INSERT INTO nuts_regions (nuts3, nuts3_name, nuts2, nuts2_name, nuts1, nuts1_name) diff --git a/scripts/normalize-raw.sql b/scripts/normalize-raw.sql index a4d49c1f6..ff5a3bc0c 100644 --- a/scripts/normalize-raw.sql +++ b/scripts/normalize-raw.sql @@ -22,8 +22,8 @@ -- * Currency is kept per row as it appears (BGN pre-2026, EUR from 2026, a few foreign) — -- NOT coerced to one currency. Money sums must group/convert by currency downstream. -- * Authorities dedupe on ЕИК (Вид на възложителя kept as `type`); a canonical name is kept. --- * Bidders dedupe on raw contractor ЕИК (kept verbatim in bulstat); is_consortium comes from --- the admin "възложена на група" flag (awarded_to_group), not a name heuristic. Resolving the +-- * Bidders use a checksum-valid contractor ЕИК, then normalised name, then one unknown bucket; +-- is_consortium comes from the entity name, while awarded_to_group stays contract-scoped. Resolving the -- members hidden behind a single consortium ЕИК needs the Търговски регистър (joined on ЕИК), -- a parked future pipeline; for now the full value is attributed to the consortium entity. -- * Tenders come from the tenders-export header row (one per УНП); lots from its lot rows. @@ -313,91 +313,157 @@ FROM raw_tenders t WHERE t.lot_id IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || t.unp); --- 4) Bidders — winning contractors, with a robust identity key: --- * valid ЕИК (digits-only, length 9/13) → key by ЕИК ('eik:') --- * otherwise (withheld 'не се публикува', foreign id, multi-ЕИК consortium, missing) → --- key by NORMALISED NAME ('name:') --- Keying invalid ЕИК by name stops the collapse where ~595 distinct withheld-ЕИК contractors --- merged onto one node. bulstat/eik_normalized stay set only for a valid ЕИК (NULL otherwise, --- which keeps the bulstat UNIQUE happy). is_consortium describes the ENTITY (a JV), so it is --- name-based — a semicolon member list or ДЗЗД / ОБЕДИНЕНИЕ / КОНСОРЦИУМ in the name; the --- per-contract awarded_to_group flag lives on contracts, not here. +-- 4) Winning-contractor identity — derive the checksum and three-rung key exactly once, then use +-- the NULL-safe raw pair for every bidder/contract projection below. +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + name_norm TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) +SELECT + eik_raw, + name_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + ELSE 'pending' + END +FROM ( + SELECT + eik_raw, + name_raw, + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') + ) +); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + +-- 4a) Bidders — valid ЕИК first, then normalised name, then one labelled unknown bucket. The +-- bucket keeps identity-poor contracts attached without polluting bulstat/eik_normalized. +-- 4a-pre) Canonical display name by frequency mode (#251), rebuilt on top of the checksum-based +-- contractor_identity keys (#252). contractor_identity is DISTINCT-ed over (eik,name), so the +-- occurrence counts the mode needs must be re-derived from raw_contracts. DROP TABLE IF EXISTS bidder_canonical_name; CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); INSERT INTO bidder_canonical_name (bidder_key, canonical_name) -SELECT bidder_key, contractor_name +SELECT bidder_key, name_raw FROM ( SELECT - bidder_key, - contractor_name, + ci.bidder_key AS bidder_key, + ci.name_raw AS name_raw, ROW_NUMBER() OVER ( - PARTITION BY bidder_key + PARTITION BY ci.bidder_key ORDER BY - cnt DESC, - CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, - LENGTH(contractor_name) DESC, - contractor_name + r.cnt DESC, + CASE WHEN ci.name_raw GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(ci.name_raw) DESC, + ci.name_raw ) AS rn - FROM ( - SELECT bidder_key, contractor_name, COUNT(*) AS cnt - FROM ( - SELECT - contractor_name, - CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key - FROM ( - SELECT - contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) - WHERE bidder_key IS NOT NULL - AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' - GROUP BY bidder_key, contractor_name - ) + FROM contractor_identity ci + JOIN ( + SELECT contractor_eik, contractor_name, COUNT(*) AS cnt + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + GROUP BY contractor_eik, contractor_name + ) r ON r.contractor_eik IS ci.eik_raw AND r.contractor_name IS ci.name_raw + WHERE ci.name_raw IS NOT NULL AND TRIM(ci.name_raw) <> '' ) WHERE rn = 1; INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - b.bidder_key, - COALESCE(bcn.canonical_name, b.bidder_key), + bidder_key, + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' + ELSE COALESCE((SELECT bcn.canonical_name FROM bidder_canonical_name bcn WHERE bcn.bidder_key = contractor_identity.bidder_key), MIN(name_raw)) END, MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END -FROM ( - SELECT - contractor_name, - eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, - CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, - CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' THEN 1 ELSE 0 - END AS grp - FROM ( - SELECT - contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) -) b -LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key -WHERE b.bidder_key IS NOT NULL -GROUP BY b.bidder_key; + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity +GROUP BY bidder_key; -- 4b) Curated public-owned winner classification. Exact EIK matches cover the allowlist; a small -- branch list handles valid 13-digit branch EIKs used by AПИ/ОПУ, ЕСО/МЕР, БНР and Информационно @@ -615,15 +681,9 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) @@ -654,6 +714,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- EOP always; an OCDS row only when no EOP row shares its contract_number - EOP wins. @@ -682,8 +744,7 @@ FROM ( ) z ) y ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key); @@ -884,15 +945,9 @@ SELECT 1, WHEN c.proc_est_eur > 0 AND c.eff_eur >= 10 * c.proc_est_eur THEN 'review' ELSE 'ok' END AS value_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + c.bidder_key FROM ( - SELECT c.*, + SELECT c.*, ci.bidder_key, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -922,6 +977,8 @@ SELECT 1, END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c -- Eligibility must mirror the INSERT INTO contracts WHERE exactly, so this candidate count is a @@ -934,8 +991,7 @@ SELECT 1, SELECT 1 FROM raw_contracts a WHERE a.source LIKE 'eop:%' AND a.contract_number = c.contract_number)) ) c - WHERE c.bidder_key IS NOT NULL - AND CASE c.value_flag + WHERE CASE c.value_flag WHEN 'annex_suspect' THEN COALESCE(c.signing_value, c.current_value) ELSE COALESCE(c.current_value, c.signing_value) END IS NOT NULL @@ -945,6 +1001,8 @@ SELECT 1, (SELECT COUNT(*) FROM contracts), datetime('now'); +DROP TABLE contractor_identity; + -- Summary (last result set printed by `wrangler d1 execute`) SELECT (SELECT COUNT(*) FROM authorities) AS authorities, diff --git a/scripts/precompute.sql b/scripts/precompute.sql index 35391e01a..34357a01a 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -158,7 +158,8 @@ SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6 FROM authority_totals at; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'contract', c.id, COALESCE(NULLIF(c.contract_subject, ''), t.title), COALESCE(t.source_id, ''), diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index 48d5fbbdf..725efdc44 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -129,80 +129,154 @@ UPDATE authorities SET type_group = CASE END WHERE type_group IS NULL; --- ── 2) Bidders referenced by OCDS staging (new ones only) — same identity rule as normalize step 4 ── +-- ── 2) Winning-contractor identity — derive the checksum and three-rung key once for this slice. ── +DROP TABLE IF EXISTS contractor_identity; +CREATE TABLE contractor_identity ( + eik_raw TEXT, + name_raw TEXT, + name_norm TEXT, + eik_clean TEXT, + eik_valid INTEGER NOT NULL, + bidder_key TEXT NOT NULL, + PRIMARY KEY (eik_raw, name_raw) +); + +INSERT INTO contractor_identity (eik_raw, name_raw, name_norm, eik_clean, eik_valid, bidder_key) +SELECT + eik_raw, + name_raw, + name_raw, + eik_clean, + eik_valid, + CASE + WHEN eik_valid = 1 THEN 'eik:' || eik_clean + ELSE 'pending' + END +FROM ( + SELECT + eik_raw, + name_raw, + eik_clean, + -- Bulgarian ЕИК/Булстат control-digit (checksum) validation. A merely SYNTACTIC 9/13-digit + -- check let the fake/service code „000000001" and wrong-digit typo twins pass, collapsing + -- unrelated foreign suppliers (Elsevier + Clarivate/Web of Science + a gas consultancy + a + -- 102 EUR construction line) onto one node (#195). Enforce the real checksum so invalid codes + -- get eik_valid = 0 and fall back to the name-based key, which splits them apart again. + -- 9-digit: weight positions 1..8 by 1..8; control = sum % 11. If that is 10, re-weight by + -- 3..10; a second 10 → 0. The 9th digit must equal the control. + -- 13-digit: the leading 9 digits must themselves be a valid 9-digit ЕИК, then weight + -- positions 9..12 by 2,7,3,5 (fallback 4,9,5,7; second 10 → 0). The 13th digit + -- must equal that control. + CASE + WHEN eik_clean IS NULL OR eik_clean IN ('000000000', '0000000000000') OR eik_clean GLOB '*[^0-9]*' OR LENGTH(eik_clean) NOT IN (9, 13) THEN 0 + WHEN ( + CASE + WHEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (1 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 2 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + WHEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 < 10 THEN (3 * CAST(SUBSTR(eik_clean, 1, 1) AS INTEGER) + 4 * CAST(SUBSTR(eik_clean, 2, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 3, 1) AS INTEGER) + 6 * CAST(SUBSTR(eik_clean, 4, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 5, 1) AS INTEGER) + 8 * CAST(SUBSTR(eik_clean, 6, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 7, 1) AS INTEGER) + 10 * CAST(SUBSTR(eik_clean, 8, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) <> CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) THEN 0 + WHEN LENGTH(eik_clean) = 9 THEN 1 + WHEN ( + CASE + WHEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (2 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 3 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + WHEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 < 10 THEN (4 * CAST(SUBSTR(eik_clean, 9, 1) AS INTEGER) + 9 * CAST(SUBSTR(eik_clean, 10, 1) AS INTEGER) + 5 * CAST(SUBSTR(eik_clean, 11, 1) AS INTEGER) + 7 * CAST(SUBSTR(eik_clean, 12, 1) AS INTEGER)) % 11 + ELSE 0 + END + ) = CAST(SUBSTR(eik_clean, 13, 1) AS INTEGER) THEN 1 + ELSE 0 + END AS eik_valid + FROM ( + SELECT + contractor_eik AS eik_raw, + contractor_name AS name_raw, + TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean + FROM (SELECT DISTINCT contractor_eik, contractor_name FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%') + ) +); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, char(160), ' '), char(9), ' '), char(10), ' '), char(13), ' '), ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '"', ''), '''', ''), '`', ''), '„', ''), '“', ''), '”', ''), '‚', ''), '‘', ''), '’', ''), '«', ''), '»', ''), '′', ''), '″', ''); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, '‐', '-'), '‑', '-'), '‒', '-'), '–', '-'), '—', '-'), '―', '-'), '−', '-'), char(173), ''), char(8203), ''); + +UPDATE contractor_identity +SET name_norm = TRIM(REPLACE(REPLACE(REPLACE(name_norm, ' ', ' '), ' ', ' '), ' ', ' ')); + +UPDATE contractor_identity +SET name_norm = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'а', 'А'), 'б', 'Б'), 'в', 'В'), 'г', 'Г'), 'д', 'Д'), 'е', 'Е'), 'ж', 'Ж'), 'з', 'З'), 'и', 'И'), 'й', 'Й'), 'к', 'К'), 'л', 'Л'), 'м', 'М'), 'н', 'Н'), 'о', 'О'); + +UPDATE contractor_identity +SET name_norm = UPPER(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(name_norm, 'п', 'П'), 'р', 'Р'), 'с', 'С'), 'т', 'Т'), 'у', 'У'), 'ф', 'Ф'), 'х', 'Х'), 'ц', 'Ц'), 'ч', 'Ч'), 'ш', 'Ш'), 'щ', 'Щ'), 'ъ', 'Ъ'), 'ь', 'Ь'), 'ю', 'Ю'), 'я', 'Я')); + +UPDATE contractor_identity +SET bidder_key = CASE + WHEN eik_valid = 1 THEN bidder_key + WHEN name_raw IS NOT NULL AND TRIM(name_raw) <> '' THEN 'name:' || name_norm + ELSE 'unknown:анонимен' +END; + +-- 4a-pre) Canonical display name by frequency mode (#251), rebuilt on top of the checksum-based +-- contractor_identity keys (#252). contractor_identity is DISTINCT-ed over (eik,name), so the +-- occurrence counts the mode needs must be re-derived from raw_contracts. DROP TABLE IF EXISTS bidder_canonical_name; CREATE TABLE bidder_canonical_name (bidder_key TEXT PRIMARY KEY, canonical_name TEXT); INSERT INTO bidder_canonical_name (bidder_key, canonical_name) -SELECT bidder_key, contractor_name +SELECT bidder_key, name_raw FROM ( SELECT - bidder_key, - contractor_name, + ci.bidder_key AS bidder_key, + ci.name_raw AS name_raw, ROW_NUMBER() OVER ( - PARTITION BY bidder_key + PARTITION BY ci.bidder_key ORDER BY - cnt DESC, - CASE WHEN contractor_name GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, - LENGTH(contractor_name) DESC, - contractor_name + r.cnt DESC, + CASE WHEN ci.name_raw GLOB '*[a-zа-я]*' THEN 0 ELSE 1 END, + LENGTH(ci.name_raw) DESC, + ci.name_raw ) AS rn - FROM ( - SELECT bidder_key, contractor_name, COUNT(*) AS cnt - FROM ( - SELECT - contractor_name, - CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key - FROM ( - SELECT - contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) - WHERE bidder_key IS NOT NULL - AND contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' - GROUP BY bidder_key, contractor_name - ) + FROM contractor_identity ci + JOIN ( + SELECT contractor_eik, contractor_name, COUNT(*) AS cnt + FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' + GROUP BY contractor_eik, contractor_name + ) r ON r.contractor_eik IS ci.eik_raw AND r.contractor_name IS ci.name_raw + WHERE ci.name_raw IS NOT NULL AND TRIM(ci.name_raw) <> '' ) WHERE rn = 1; INSERT OR IGNORE INTO bidders (id, name, bulstat, eik_normalized, eik_valid, is_consortium, kind) SELECT - b.bidder_key, - COALESCE(bcn.canonical_name, b.bidder_key), + bidder_key, + CASE WHEN bidder_key = 'unknown:анонимен' THEN 'Неизвестен изпълнител' + ELSE COALESCE((SELECT bcn.canonical_name FROM bidder_canonical_name bcn WHERE bcn.bidder_key = contractor_identity.bidder_key), MIN(name_raw)) END, MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MIN(CASE WHEN eik_valid = 1 THEN eik_clean END), MAX(eik_valid), - MAX(grp), - CASE WHEN MAX(grp) = 1 THEN 'consortium' ELSE 'company' END -FROM ( - SELECT contractor_name, eik_clean, - CASE WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 1 ELSE 0 END AS eik_valid, - CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key, - CASE - WHEN contractor_name LIKE '%;%' - OR UPPER(contractor_name) LIKE '%ДЗЗД%' - OR UPPER(contractor_name) LIKE '%ОБЕДИНЕНИЕ%' - OR UPPER(contractor_name) LIKE '%КОНСОРЦИУМ%' + MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' + THEN 1 ELSE 0 + END), + CASE + WHEN bidder_key = 'unknown:анонимен' THEN 'unknown' + WHEN MAX(CASE + WHEN name_raw LIKE '%;%' + OR UPPER(name_raw) LIKE '%ДЗЗД%' + OR UPPER(name_raw) LIKE '%ОБЕДИНЕНИЕ%' + OR UPPER(name_raw) LIKE '%КОНСОРЦИУМ%' THEN 1 ELSE 0 - END AS grp - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) -) b -LEFT JOIN bidder_canonical_name bcn ON bcn.bidder_key = b.bidder_key -WHERE b.bidder_key IS NOT NULL -GROUP BY b.bidder_key + END) = 1 THEN 'consortium' + ELSE 'company' + END +FROM contractor_identity +GROUP BY bidder_key ON CONFLICT(id) DO UPDATE SET name = excluded.name, bulstat = COALESCE(bidders.bulstat, excluded.bulstat), @@ -803,16 +877,10 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + c.bidder_key FROM ( - SELECT c.*, - c.authority_name AS authority_name_raw, + SELECT c.*, ci.bidder_key, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -842,6 +910,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'ocds:%' @@ -849,8 +919,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) -- EOP wins: skip OCDS rows when the transient window has an EOP row for the same document. @@ -1053,16 +1122,10 @@ FROM ( AND c.contract_date > date(c.published_at, '+2 day') THEN 'signed_after_publication' ELSE 'ok' END AS date_flag, - CASE - WHEN TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) NOT GLOB '*[^0-9]*' - AND LENGTH(TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END)) IN (9, 13) - THEN 'eik:' || TRIM(CASE WHEN c.contractor_eik LIKE 'ЕИК %' THEN SUBSTR(c.contractor_eik, 5) ELSE c.contractor_eik END) - WHEN c.contractor_name IS NOT NULL AND TRIM(c.contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(c.contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key + c.bidder_key FROM ( - SELECT c.*, - c.authority_name AS authority_name_raw, + SELECT c.*, ci.bidder_key, + c.authority_name AS authority_name_raw, CASE WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'EUR' THEN COALESCE(c.current_value, c.signing_value) WHEN COALESCE(NULLIF(c.currency, ''), 'BGN') = 'BGN' THEN COALESCE(c.current_value, c.signing_value) / 1.95583 @@ -1092,6 +1155,8 @@ FROM ( END AS proc_est_eur, t.estimated_value AS proc_est_native FROM raw_contracts c + JOIN contractor_identity ci + ON c.contractor_eik IS ci.eik_raw AND c.contractor_name IS ci.name_raw LEFT JOIN tenders t ON t.id = 't:' || c.unp ) c WHERE c.source LIKE 'eop:%' @@ -1111,8 +1176,7 @@ FROM ( ) y ) q ) x -WHERE x.bidder_key IS NOT NULL - AND x.display_native IS NOT NULL +WHERE x.display_native IS NOT NULL AND EXISTS (SELECT 1 FROM tenders te WHERE te.id = 't:' || x.unp) AND EXISTS (SELECT 1 FROM bidders b WHERE b.id = x.bidder_key) AND NOT EXISTS ( @@ -1368,23 +1432,9 @@ INSERT OR IGNORE INTO refresh_touched_bidders (bidder_id) SELECT b.id FROM bidders b WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NULL) - OR b.id IN ( - SELECT bidder_key - FROM ( - SELECT contractor_name, eik_clean, - CASE - WHEN eik_clean NOT GLOB '*[^0-9]*' AND LENGTH(eik_clean) IN (9, 13) THEN 'eik:' || eik_clean - WHEN contractor_name IS NOT NULL AND TRIM(contractor_name) <> '' THEN 'name:' || UPPER(TRIM(REPLACE(REPLACE(contractor_name, ' ', ' '), ' ', ' '))) - ELSE NULL - END AS bidder_key - FROM ( - SELECT contractor_name, - TRIM(CASE WHEN contractor_eik LIKE 'ЕИК %' THEN SUBSTR(contractor_eik, 5) ELSE contractor_eik END) AS eik_clean - FROM raw_contracts WHERE source LIKE 'eop:%' OR source LIKE 'ocds:%' - ) - ) - WHERE bidder_key IS NOT NULL - ); + OR b.id IN (SELECT bidder_key FROM contractor_identity); + +DROP TABLE contractor_identity; -- 6) Refresh rollups + FTS. Only the D1-hot rollups are scoped to touched rows; cheaper rollups stay -- full-recomputed in isolated batches so convergence stays simple. @@ -1430,7 +1480,8 @@ GROUP BY t.authority_id, c.bidder_id; DELETE FROM search_index WHERE kind = 'company'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'company', ct.bidder_id, ct.name, COALESCE(ct.eik, ''), COALESCE(ct.settlement, ''), ct.won_eur -FROM company_totals ct; +FROM company_totals ct +WHERE ct.bidder_id <> 'unknown:анонимен'; DELETE FROM search_index WHERE kind = 'authority'; INSERT INTO search_index (kind, ref, title, ident, subtitle, amount) SELECT 'authority', at.authority_id, at.name, COALESCE(substr(at.authority_id, 6), ''), COALESCE(at.settlement, ''), at.spent_eur