Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
fc1878f
fix(etl): enforce bulstat checksum for eik identity
todorkolev Jul 17, 2026
ce14433
feat(etl): canonical entity fields by frequency mode, not MIN/MAX
todorkolev Jul 17, 2026
5fa16cc
feat(db): attribute joint procurements to each co-authority
todorkolev Jul 17, 2026
45b100a
feat(etl,web): preserve verbatim ordering-unit name beside the canoni…
todorkolev Jul 17, 2026
897a421
fix(etl,web): preserve identity-poor contracts via an unknown-bidder …
todorkolev Jul 18, 2026
efb1091
fix(etl): fold contractor name-key over case, quotes and dash variants
todorkolev Jul 19, 2026
c10a897
feat(db): pick joint leads by УНП prefix and value the participation
todorkolev Jul 20, 2026
48e3302
fix(db): exclude the unknown-bidder bucket from authority top-suppliers
todorkolev Jul 22, 2026
d9e291e
fix(etl): exclude composite joint EIKs from authority minting
todorkolev Jul 23, 2026
ddedd0a
fix(etl): skip undecomposable composite EIKs when seeding joint members
todorkolev Jul 23, 2026
f773680
merge main (#246 #263 #156 #259)
todorkolev Jul 23, 2026
ebb0081
merge main (#260)
Jul 23, 2026
65463fc
merge #251 (canonical identity + ordering_unit_name + composite guard)
todorkolev Jul 23, 2026
8a150c4
merge main (#251 squashed; identical content, ours carries the +#252 …
todorkolev Jul 23, 2026
7ed5a4c
merge #252 resolution (contractor_identity + canonical identity)
todorkolev Jul 23, 2026
0a35228
style: prettier-format companies.test.ts
todorkolev Jul 23, 2026
38379c5
merge main (#252 squashed; ours carries the +#253 resolution)
todorkolev Jul 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions packages/db/migrations/0000_init.sql
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,17 @@ CREATE TABLE contracts (
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);

-- Joint procurements retain one lead authority on tenders (the only authority that receives spend),
-- while this bridge records every participating authority. ordinal 0 is always the lead.
CREATE TABLE contract_co_authorities (
contract_id TEXT NOT NULL REFERENCES contracts(id) ON DELETE CASCADE,
authority_id TEXT NOT NULL REFERENCES authorities(id),
ordinal INTEGER NOT NULL CHECK (ordinal >= 0),
PRIMARY KEY (contract_id, authority_id)
);
CREATE INDEX idx_contract_co_authorities_authority
ON contract_co_authorities(authority_id, contract_id);

-- Domain amendment history used to roll current_value/annex_count without staging. Natural-keyed so
-- re-imports are idempotent; built by scripts/promote-amendments.sql from the transient amendment feed.
CREATE TABLE amendments (
Expand Down Expand Up @@ -246,6 +257,15 @@ CREATE TABLE authority_totals (
last_date TEXT
);

-- Joint-procurement participation rollup. joint_contract_value_eur is the total value of the joint
-- procurements the authority took part in; it is informational only and must NEVER be summed into
-- authority_totals.spent_eur or any national/leaderboard total.
CREATE TABLE authority_joint_participation (
authority_id TEXT PRIMARY KEY REFERENCES authorities(id),
joint_contract_participations INTEGER NOT NULL,
joint_contract_value_eur REAL NOT NULL DEFAULT 0
);

-- Per CPV division. Sector facet + filter counts on the list pages.
CREATE TABLE sector_totals (
division TEXT PRIMARY KEY, -- 2-digit CPV division (joins @sigma/config CPV_SECTORS)
Expand Down
13 changes: 13 additions & 0 deletions packages/db/src/migrations.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,19 @@ describe('served migrations', () => {
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='amendments';",
).trim(),
).toBe('1');

expect(
sqlite(
dbPath,
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='contract_co_authorities';",
).trim(),
).toBe('1');
expect(
sqlite(
dbPath,
"SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='authority_joint_participation';",
).trim(),
).toBe('1');
expect(
sqlite(
dbPath,
Expand Down
135 changes: 135 additions & 0 deletions packages/db/src/refresh-slice.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql');
const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql');
const normalizePath = resolve(root, 'scripts/normalize-raw.sql');
const precomputePath = resolve(root, 'scripts/precompute.sql');
const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql');

function sqlite(dbPath: string, sql: string): string {
Expand Down Expand Up @@ -352,6 +353,140 @@ function seedTwoCollidingBidders(dbPath: string): void {
}
}

function seedJointAuthorityFixture(dbPath: string, refresh: boolean): void {
if (refresh) {
sqlite(
dbPath,
`INSERT INTO authorities (id, name, bulstat, type)
VALUES ('auth:222222222', 'Second Authority', '222222222', 'public');
INSERT INTO tenders
(id, source_id, title, authority_id, currency, procedure_type, status)
VALUES
('t:00080-2023-0001', '00080-2023-0001', 'Historical prefix observation',
'auth:222222222', 'EUR', 'open', 'published');`,
);
}

sqlite(
dbPath,
`INSERT INTO raw_tenders
(source, dataset_year, fetched_at, unp, tender_id, procedure_type, procurement_subject,
estimated_value, currency, authority_name, authority_eik, authority_type, lot_id, published_at)
VALUES
('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2023-0001',
'TENDER-PREFIX-HISTORY', 'open', 'Historical prefix observation', 1000, 'EUR',
'Second Authority', '222222222', 'public', NULL, '2023-01-01'),
('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '99999-2023-0001',
'TENDER-NAME-HISTORY', 'open', 'Historical name observation', 1000, 'EUR',
'First Authority', '111111111', 'public', NULL, '2023-01-01'),
('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', '00080-2024-0030',
'TENDER-JOINT-PREFIX', 'open', 'Prefix-led joint tender', 1000, 'EUR',
'First Authority', '111111111;222222222', 'public', NULL, '2024-01-01'),
('eop:tenders:2024-01-01', 2024, '2024-01-02T00:00:00Z', 'UNRESOLVED-2024-0001',
'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR',
'Third Authority', '333333333;444444444', 'public', NULL, '2024-01-01');

INSERT INTO raw_contracts
(source, dataset_year, dataset_variant, fetched_at, needs_enrichment, document_number,
published_at, unp, tender_ext_id, procedure_type, procurement_subject, estimated_value,
procurement_currency, authority_name, authority_eik, authority_type, contract_number,
contract_date, signing_value, currency, contract_subject, awarded_to_group,
contractor_eik, contractor_name, eu_funded, bids_received)
VALUES
('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0,
'DOC-JOINT-PREFIX', '2024-01-01', '00080-2024-0030', 'TENDER-JOINT-PREFIX', 'open',
'Prefix-led joint tender', 1000, 'EUR', 'First Authority', '111111111;222222222',
'public', 'CONTRACT-JOINT-PREFIX', '2024-01-02', 100, 'EUR', 'Prefix-led contract', 0,
'777777777', 'Joint Bidder', 0, 1),
('eop:contracts:2024-01-01', 2024, 'eop', '2024-01-02T00:00:00Z', 0,
'DOC-JOINT-FALLBACK', '2024-01-01', 'UNRESOLVED-2024-0001',
'TENDER-JOINT-FALLBACK', 'open', 'Fallback joint tender', 1000, 'EUR',
'Third Authority', '333333333;444444444', 'public', 'CONTRACT-JOINT-FALLBACK',
'2024-01-02', 200, 'EUR', 'Fallback contract', 0, '777777777', 'Joint Bidder', 0, 1);`,
);
}

describe('joint-procurement authority attribution', () => {
it.each([
['full normalize', false],
['slice refresh', true],
] as const)(
'uses the УНП prefix and keeps participation value non-summable on %s',
(_label, refresh) => {
const dir = mkdtempSync(resolve(tmpdir(), 'sigma-joint-authorities-'));
const dbPath = resolve(dir, 'test.sqlite');
try {
initWorkDb(dbPath);
seedJointAuthorityFixture(dbPath, refresh);
readScript(dbPath, refresh ? refreshSlicePath : normalizePath);
if (!refresh) readScript(dbPath, precomputePath);

expect(
sqliteJson<{ source_id: string; authority_id: string }>(
dbPath,
`SELECT source_id, authority_id FROM tenders
WHERE source_id IN ('00080-2024-0030', 'UNRESOLVED-2024-0001')
ORDER BY source_id`,
),
).toEqual([
{ source_id: '00080-2024-0030', authority_id: 'auth:222222222' },
{ source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333' },
]);

expect(
sqliteJson<{ source_id: string; authority_id: string; ordinal: number }>(
dbPath,
`SELECT t.source_id, cca.authority_id, cca.ordinal
FROM contract_co_authorities cca
JOIN contracts c ON c.id = cca.contract_id
JOIN tenders t ON t.id = c.tender_id
WHERE cca.ordinal = 0
ORDER BY t.source_id`,
),
).toEqual([
{ source_id: '00080-2024-0030', authority_id: 'auth:222222222', ordinal: 0 },
{ source_id: 'UNRESOLVED-2024-0001', authority_id: 'auth:333333333', ordinal: 0 },
]);

expect(
sqliteJson<{
authority_id: string;
joint_contract_participations: number;
joint_contract_value_eur: number;
}>(
dbPath,
`SELECT authority_id, joint_contract_participations, joint_contract_value_eur
FROM authority_joint_participation
WHERE authority_id IN ('auth:111111111', 'auth:222222222')
ORDER BY authority_id`,
),
).toEqual([
{
authority_id: 'auth:111111111',
joint_contract_participations: 1,
joint_contract_value_eur: 100,
},
{
authority_id: 'auth:222222222',
joint_contract_participations: 1,
joint_contract_value_eur: 100,
},
]);

expect(
sqliteJson<{ spent_eur: number }>(
dbPath,
`SELECT COALESCE((SELECT spent_eur FROM authority_totals
WHERE authority_id = 'auth:111111111'), 0) AS spent_eur`,
)[0]?.spent_eur,
).toBe(0);
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
);
});

describe('refresh-slice EOP base derivation', () => {
it('derives new eop base rows as c:e contracts and is idempotent', () => {
const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-'));
Expand Down
Loading