diff --git a/apps/web/app/routes/contract.tsx b/apps/web/app/routes/contract.tsx index c9450cc63..f3e393c56 100644 --- a/apps/web/app/routes/contract.tsx +++ b/apps/web/app/routes/contract.tsx @@ -1,7 +1,7 @@ import { Link } from 'react-router'; import { count, longDate, money, moneyBare, plural, signedMoney, signedPct } from '@sigma/shared'; import { contractIdFromSlug, contractSlug, getContract, getDb } from '@sigma/db'; -import type { ContractDetail } from '@sigma/api-contract'; +import type { CohortBand, ContractDetail } from '@sigma/api-contract'; import type { Route } from './+types/contract'; import { Breadcrumbs } from '../components/Breadcrumbs'; import { PageHeader } from '../components/PageHeader'; @@ -92,10 +92,24 @@ export async function loader({ params, context }: Route.LoaderArgs) { return { contract }; } +// Coarse cohort bands only (never a fake-precise "топ 4.7%") - @sigma/db cohortBand only claims a +// fine band when the cohort is large enough and its percentiles are distinct (see cohort.ts). +const COHORT_BAND_LABELS: Record = { + top1: 'в най-горния 1% по стойност', + top5: 'в топ 5% по стойност', + top10: 'в топ 10% по стойност', + top25: 'в топ 25% по стойност', + 'above-median': 'над медианата', + 'at-median': 'около медианата', + 'below-median': 'под медианата', + bottom25: 'сред най-ниските 25% по стойност', +}; + const UNVERIFIED_VALUE_LABEL = 'стойност с непотвърдена достоверност'; export default function Contract({ loaderData }: Route.ComponentProps) { const c = loaderData.contract; + const cohort = c.cohort; const v = c.value; const crumbId = c.unp || c.contractNumber || c.id; // Direct links to the day's raw ЦАИС ЕОП open-data files (storage.eop.bg) this record was @@ -356,6 +370,34 @@ export default function Contract({ loaderData }: Route.ComponentProps) { + {cohort && ( +
+

+ {money(cohort.amountEur)} е{' '} + {COHORT_BAND_LABELS[cohort.band]} сред{' '} + {count(cohort.stats.pricedContracts)}{' '} + {plural(cohort.stats.pricedContracts, 'договор', 'договора')} в сектор „ + {c.sector?.short ?? `CPV ${cohort.stats.division}`}“ (CPV {cohort.stats.division}). + Медианата за сектора е {money(cohort.stats.medianEur)}. +

+

+ Приблизителна позиция по предизчислени персентили на сектора, включващи и самия този + договор. Сравнението дава контекст на мащаба и не е оценка за нередност - голяма + поръчка може да е напълно обоснована. Използва различен метод от отчета за аномалии, + затова числата може леко да се разминават. +

+

+ + Виж договорите в сектора → + +

+
+ )} +
+

+ „Подобни договори" на страницата на договор. Позицията („топ 5%", + „над медианата"…) сравнява стойността с всички договори с чиста стойност в същия + двуцифрен CPV сектор в базата (без времева граница - същият обхват като секторните + суми), по предизчислени персентили на сектора (включващи и самия договор). Показваме + сравнение само при поне 12 такива договора, а по-фина лента („топ 1%/5%/10%") — само + когато секторът е достатъчно голям и съответните персентили реално се различават, за + да не се появи фалшива точност при малка или изравнена по цена кохорта. Позицията е + приблизителна (широки стъпала, не точен процент), контекст за мащаба, а не оценка за + нередност, и ползва различен метод от отчета за аномалии, затова двете числа може + леко да се разминават. Договори с непотвърдена стойност не участват и не получават + сравнение. +

diff --git a/docs/etl-pipeline-state.md b/docs/etl-pipeline-state.md index c01286ba8..1801f899b 100644 --- a/docs/etl-pipeline-state.md +++ b/docs/etl-pipeline-state.md @@ -33,7 +33,7 @@ Scoped, идемпотентен дневен delta refresh. Заменя сам **Записва директно в live (served) таблици — няма blue-green swap:** - contracts (DELETE стари `c:e:`/`c:o:` от прозореца → INSERT нови с `amount_eur`) -- `authority_totals`, `company_totals`, `flow_pairs`, `home_totals`, `sector_totals`, `facet_counts` (DELETE + REPLACE на засегнатите rollup-и) +- `authority_totals`, `company_totals`, `flow_pairs`, `home_totals`, `sector_totals`, `facet_counts`, `cpv_division_stats` (DELETE + REPLACE на засегнатите rollup-и) - `search_index`, `data_freshness` **Извиква се от:** @@ -99,7 +99,7 @@ Gate-ът е вързан **само в operator скриптовете** (`impo | Rollup | Поведение | Глобална консистентност | |--------|-----------|--------------------------| | `company_totals`, `authority_totals` | **scoped** към touched множеството (`refresh-slice.sql:1262`) | зависи от touched множеството | -| `home_totals`, `sector_totals`, `facet_counts`, `flow_pairs`, `data_freshness` | **full-recompute** всеки run | по конструкция ✅ | +| `home_totals`, `sector_totals`, `facet_counts`, `flow_pairs`, `cpv_division_stats`, `data_freshness` | **full-recompute** всеки run | по конструкция ✅ | Touched множеството се строи от **новата** атрибуция (`refresh-slice.sql:1198–1239`), след DELETE+INSERT на договорите. Contract id-то вгражда `bidder_key` (`refresh-slice.sql:527`), а DELETE-ът мачва само по `contract_number + tender` (ред 499). diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index 4d8408676..43c16b3ae 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -332,6 +332,10 @@ export interface ContractDetail { lots: ContractLots | null; /** Declared subcontractor from the АОП feed ("Подизпълнител"), sparse (~0.8% of contracts). */ subcontractor: { name: string; eik: string | null; valueEur: number | null } | null; + /** „Подобни договори" value benchmark vs the contract's CPV-division cohort — null when no honest + * comparison exists (suspect/absent value, no CPV, or a cohort below the minimum). Computed inside + * getContract from the row it already read, so it costs one extra rollup read, not a second scan. */ + cohort: ContractCohortBenchmark | null; /** Published amendments (annexes), oldest first — the recorded value history behind * `value.signingEur` → `value.currentEur`. Empty when the contract has no annexes. */ amendments: AmendmentEntry[]; @@ -342,6 +346,38 @@ export interface ContractRecord extends ContractDetail { sourceNames: { authority: string; bidder: string }; // verbatim source names } +/** Precomputed value percentiles of one CPV division (cpv_division_stats rollup). */ +export interface CpvCohortStats { + division: string; + pricedContracts: number; + p25Eur: number; + medianEur: number; + p75Eur: number; + p90Eur: number; + p95Eur: number; + p99Eur: number; +} + +/** Coarse position of one contract's value inside its CPV-division cohort. A „top X%" band is only + * claimed when the cohort is large enough for that cut to be real AND the percentile anchors around + * it are distinct (so a tie-collapsed or tiny cohort never yields a fake „top 1%"). */ +export type CohortBand = + | 'top1' + | 'top5' + | 'top10' + | 'top25' + | 'above-median' + | 'at-median' + | 'below-median' + | 'bottom25'; + +/** The „Подобни договори" benchmark for the contract page - null when there is no honest cohort. */ +export interface ContractCohortBenchmark { + amountEur: number; + band: CohortBand; + stats: CpvCohortStats; +} + // ── Flows ─────────────────────────────────────────────────────────────────────────────────────── export interface FlowPair { diff --git a/packages/db/migrations/0004_cpv_division_stats.sql b/packages/db/migrations/0004_cpv_division_stats.sql new file mode 100644 index 000000000..7dfbca79e --- /dev/null +++ b/packages/db/migrations/0004_cpv_division_stats.sql @@ -0,0 +1,13 @@ +-- Per-CPV-division value percentiles for the contract-page "Подобни договори" benchmark. +-- Filled by scripts/precompute.sql (full rebuild) and scripts/refresh-slice.sql (cron refresh); +-- the contract page reads ONE row here instead of scanning its whole division per view. +CREATE TABLE IF NOT EXISTS cpv_division_stats ( + division TEXT PRIMARY KEY, + priced_contracts INTEGER NOT NULL, + p25_eur REAL NOT NULL, + median_eur REAL NOT NULL, + p75_eur REAL NOT NULL, + p90_eur REAL NOT NULL, + p95_eur REAL NOT NULL, + p99_eur REAL NOT NULL +); diff --git a/packages/db/src/precompute-cohort.test.ts b/packages/db/src/precompute-cohort.test.ts new file mode 100644 index 000000000..2c1c67639 --- /dev/null +++ b/packages/db/src/precompute-cohort.test.ts @@ -0,0 +1,135 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +// Apply EVERY migration in the directory, not a hardcoded subset: precompute.sql reads columns added +// by later migrations (current_value_currency from 0002), so a pinned list silently drifts out of the +// real served schema and fails on a column the production DB has. Sorted, so ordering stays by number. +const migrationsDir = resolve(root, 'packages/db/migrations'); +const migrations = readdirSync(migrationsDir) + .filter((f) => f.endsWith('.sql')) + .sort() + .map((f) => resolve(migrationsDir, f)); +const precomputePath = resolve(root, 'scripts/precompute.sql'); +const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); + +function sqlite(dbPath: string, sql: string): string { + return execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', ['-bail', dbPath], { input: `.read ${path}\n`, stdio: 'pipe' }); +} + +interface StatsRow { + division: string; + priced_contracts: number; + p25_eur: number; + median_eur: number; + p75_eur: number; + p90_eur: number; + p95_eur: number; + p99_eur: number; +} + +describe('precompute cpv_division_stats', () => { + it('computes nearest-rank percentiles per division over the clean-value cohort', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-cohort-')); + const dbPath = resolve(dir, 'test.sqlite'); + try { + for (const migration of migrations) readScript(dbPath, migration); + + // Division 45: 20 clean contracts valued 1000..20000 EUR, plus rows the cohort must exclude + // (suspect flag, NULL amount, zero amount, missing CPV). Division 30: a small 3-row cohort + // still gets a stats row - the MIN_COHORT floor is applied read-side, not here. + const stmts: string[] = [ + `INSERT INTO authorities (id, name) VALUES ('auth:1', 'Възложител');`, + `INSERT INTO bidders (id, name) VALUES ('eik:1', 'Изпълнител');`, + `INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type) + VALUES ('t:45', 'UNP-45', 'Строителство', 'auth:1', '45000000', 'открита процедура'), + ('t:30', 'UNP-30', 'Техника', 'auth:1', '30200000', 'открита процедура'), + ('t:none', 'UNP-NONE', 'Без CPV', 'auth:1', NULL, 'открита процедура');`, + ]; + for (let i = 1; i <= 20; i += 1) { + stmts.push( + `INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, value_flag) + VALUES ('c:45-${i}', 't:45', 'eik:1', ${i * 1000}, ${i * 1000}, 'ok');`, + ); + } + stmts.push( + `INSERT INTO contracts (id, tender_id, bidder_id, amount, amount_eur, value_flag) + VALUES ('c:45-suspect', 't:45', 'eik:1', 9e9, 9e9, 'value_suspect'), + ('c:45-null', 't:45', 'eik:1', 5, NULL, 'ok'), + ('c:45-zero', 't:45', 'eik:1', 0, 0, 'ok'), + ('c:no-cpv', 't:none', 'eik:1', 7000, 7000, 'ok'), + ('c:30-1', 't:30', 'eik:1', 100, 100, 'ok'), + ('c:30-2', 't:30', 'eik:1', 200, 200, 'ok'), + ('c:30-3', 't:30', 'eik:1', 300, 300, 'ok');`, + ); + sqlite(dbPath, stmts.join('\n')); + + readScript(dbPath, precomputePath); + + const rows = sqliteJson( + dbPath, + 'SELECT * FROM cpv_division_stats ORDER BY division;', + ); + expect(rows).toHaveLength(2); + + // n=20 → nearest-rank k = ceil(q*20): p25→5th, p50→10th, p75→15th, p90→18th, p95→19th, p99→20th. + expect(rows[1]).toEqual({ + division: '45', + priced_contracts: 20, + p25_eur: 5000, + median_eur: 10000, + p75_eur: 15000, + p90_eur: 18000, + p95_eur: 19000, + p99_eur: 20000, + }); + + // n=3 → k = ceil(q*3): p25→1st, p50→2nd, p75/p90/p95/p99→3rd. + expect(rows[0]).toEqual({ + division: '30', + priced_contracts: 3, + p25_eur: 100, + median_eur: 200, + p75_eur: 300, + p90_eur: 300, + p95_eur: 300, + p99_eur: 300, + }); + + // Idempotent: a re-run reflects current rows, no duplicates or drift. + readScript(dbPath, precomputePath); + expect( + sqlite(dbPath, "SELECT COUNT(*) FROM cpv_division_stats WHERE division = '45';").trim(), + ).toBe('1'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('keeps the refresh-slice rebuild identical to the precompute one', () => { + // Both files carry a full-rebuild copy of the same statement (small global rollup, like + // sector_totals). Extract INSERT INTO cpv_division_stats ... ; from each and compare with + // whitespace normalised, so the two paths cannot silently drift apart. + const extract = (path: string): string => { + const sql = readFileSync(path, 'utf8'); + const m = /INSERT INTO cpv_division_stats[\s\S]*?;/.exec(sql); + if (!m) throw new Error(`no cpv_division_stats insert in ${path}`); + return m[0].replace(/\s+/g, ' ').trim(); + }; + expect(extract(refreshSlicePath)).toBe(extract(precomputePath)); + }); +}); diff --git a/packages/db/src/queries/cohort.test.ts b/packages/db/src/queries/cohort.test.ts new file mode 100644 index 000000000..42b8015b7 --- /dev/null +++ b/packages/db/src/queries/cohort.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest'; +import type { CpvCohortStats } from '@sigma/api-contract'; +import { MIN_COHORT, cohortBand, contractCohort, getCpvCohortStats } from './cohort'; + +// A large, well-separated cohort — every band is reachable here. +const stats: CpvCohortStats = { + division: '45', + pricedContracts: 200, + p25Eur: 10_000, + medianEur: 50_000, + p75Eur: 120_000, + p90Eur: 400_000, + p95Eur: 900_000, + p99Eur: 4_000_000, +}; + +describe('cohortBand — happy path (large, separated cohort)', () => { + it('maps an amount onto the band ladder', () => { + expect(cohortBand(5_000_000, stats)).toBe('top1'); + expect(cohortBand(1_000_000, stats)).toBe('top5'); + expect(cohortBand(500_000, stats)).toBe('top10'); + expect(cohortBand(200_000, stats)).toBe('top25'); + expect(cohortBand(60_000, stats)).toBe('above-median'); + expect(cohortBand(20_000, stats)).toBe('below-median'); + expect(cohortBand(500, stats)).toBe('bottom25'); + }); +}); + +// Regression for nedda76's Request-Changes findings #1–#3: the band must never over-claim precision +// the shared, self-inclusive percentile grid cannot support. +describe('cohortBand — no false precision', () => { + it('#1 a value EQUAL to an anchor does not qualify for that top band (strict >)', () => { + // Exactly p99 is not "top 1%" — with ties, the anchor value is shared by many rows. + expect(cohortBand(stats.p99Eur, stats)).toBe('top5'); // > p95, but not > p99 + expect(cohortBand(stats.p95Eur, stats)).toBe('top10'); // > p90, but not > p95 + expect(cohortBand(stats.p75Eur, stats)).toBe('above-median'); + }); + + it('#1 a fully tie-collapsed cohort (all one price) never yields "top 1%"', () => { + // 200 framework contracts all at 100 000 € → every percentile equals 100 000. + const flat: CpvCohortStats = { + division: '45', + pricedContracts: 200, + p25Eur: 100_000, + medianEur: 100_000, + p75Eur: 100_000, + p90Eur: 100_000, + p95Eur: 100_000, + p99Eur: 100_000, + }; + // Every such contract reads as "at median", not "top 1%". + expect(cohortBand(100_000, flat)).toBe('at-median'); + }); + + it('#2 the most-expensive contract in a small cohort is NOT labelled "top 1%"', () => { + // N = 12 is above MIN_COHORT but far below the top1/top5/top10 floors; only top25 is offered. + const small: CpvCohortStats = { ...stats, pricedContracts: 12 }; + expect(cohortBand(9_999_999, small)).toBe('top25'); // > p75, best band a 12-row cohort earns + const mid: CpvCohortStats = { ...stats, pricedContracts: 40 }; + expect(cohortBand(9_999_999, mid)).toBe('top5'); // 40 rows unlock top5 but not top1 (needs 100) + }); + + it('#3 the nearest-rank median value is "at-median", not "above-median"', () => { + expect(cohortBand(stats.medianEur, stats)).toBe('at-median'); + expect(cohortBand(stats.medianEur + 1, stats)).toBe('above-median'); + expect(cohortBand(stats.medianEur - 1, stats)).toBe('below-median'); + }); + + it('skips a fine band when its anchor is not strictly above the coarser one', () => { + // p99 == p95 (a flat top): a value above them is NOT "top 1%" — the anchor does not separate the + // band — it falls to the coarser top5 (whose p95>p90 guard still holds). + const flatTop: CpvCohortStats = { ...stats, p95Eur: 4_000_000, p99Eur: 4_000_000 }; + expect(cohortBand(5_000_000, flatTop)).toBe('top5'); + }); +}); + +interface FakeStats { + stats?: Partial | null; +} + +function fakeDb({ stats: statsOverride = null }: FakeStats): D1Database { + const statsRow = + statsOverride === null + ? null + : { + division: statsOverride.division ?? '45', + priced_contracts: statsOverride.pricedContracts ?? 200, + p25_eur: statsOverride.p25Eur ?? 10_000, + median_eur: statsOverride.medianEur ?? 50_000, + p75_eur: statsOverride.p75Eur ?? 120_000, + p90_eur: statsOverride.p90Eur ?? 400_000, + p95_eur: statsOverride.p95Eur ?? 900_000, + p99_eur: statsOverride.p99Eur ?? 4_000_000, + }; + return { + prepare(_sql: string) { + return { + bind() { + return this; + }, + async first() { + return statsRow as T; + }, + }; + }, + } as unknown as D1Database; +} + +describe('getCpvCohortStats', () => { + it('maps the rollup row to the DTO', async () => { + expect(await getCpvCohortStats(fakeDb({ stats: {} }), '45')).toEqual(stats); + }); + it('returns null for a division without a rollup row', async () => { + expect(await getCpvCohortStats(fakeDb({}), '99')).toBeNull(); + }); +}); + +describe('contractCohort — pure, no second contract read', () => { + it('returns the benchmark for a clean-value contract in a big-enough cohort', () => { + const b = contractCohort(1_000_000, 'ok', '45', stats); + expect(b).not.toBeNull(); + expect(b?.amountEur).toBe(1_000_000); + expect(b?.band).toBe('top5'); + expect(b?.stats.division).toBe('45'); + }); + + it('returns null without a CPV division', () => { + expect(contractCohort(1_000_000, 'ok', '', stats)).toBeNull(); + }); + + it('returns null without a clean value (suspect flag, null or zero amount)', () => { + expect(contractCohort(1_000_000, 'value_suspect', '45', stats)).toBeNull(); + expect(contractCohort(null, 'ok', '45', stats)).toBeNull(); + expect(contractCohort(0, 'ok', '45', stats)).toBeNull(); + }); + + it('returns null when there are no stats or the cohort is below MIN_COHORT', () => { + expect(contractCohort(1_000_000, 'ok', '45', null)).toBeNull(); + expect( + contractCohort(1_000_000, 'ok', '45', { ...stats, pricedContracts: MIN_COHORT - 1 }), + ).toBeNull(); + }); +}); diff --git a/packages/db/src/queries/cohort.ts b/packages/db/src/queries/cohort.ts new file mode 100644 index 000000000..5cf186adb --- /dev/null +++ b/packages/db/src/queries/cohort.ts @@ -0,0 +1,102 @@ +// „Подобни договори" - the contract page's CPV-cohort value benchmark. Reads the precomputed +// cpv_division_stats rollup (scripts/precompute.sql / refresh-slice.sql) with one O(1) PK read; the +// per-contract fields come from the row getContract already fetched, so there is no second scan. + +import type { CohortBand, ContractCohortBenchmark, CpvCohortStats } from '@sigma/api-contract'; + +/** + * Minimum priced contracts before a division cohort is shown at all - mirrors the anomaly report's + * cohort floor (scripts/anomaly-report.mjs, ANOMALY_DEFAULTS.minCohort). The finer „top X%" bands + * require larger cohorts still (see BAND_MIN_COHORT). + */ +export const MIN_COHORT = 12; + +// A „top X%" band is only honest when the cohort has enough rows that the cut names a real fraction. +// nearest-rank pXX of N rows is the ceil(XX·N)-th smallest, so „strictly above pXX" first isolates a +// single row at exactly these sizes (top1 → the 1 largest of 100; top5 → top 2 of 40; top10 → top 2 +// of 20; top25 → top 3 of 12). Below the threshold the band is not offered and we fall back coarser. +const BAND_MIN_COHORT = { top1: 100, top5: 40, top10: 20, top25: 12 } as const; + +interface CohortStatsRow { + division: string; + priced_contracts: number; + p25_eur: number; + median_eur: number; + p75_eur: number; + p90_eur: number; + p95_eur: number; + p99_eur: number; +} + +export async function getCpvCohortStats( + db: D1Database, + division: string, +): Promise { + const row = await db + .prepare(`SELECT * FROM cpv_division_stats WHERE division = ?`) + .bind(division) + .first(); + if (!row) return null; + return { + division: row.division, + pricedContracts: row.priced_contracts, + p25Eur: row.p25_eur, + medianEur: row.median_eur, + p75Eur: row.p75_eur, + p90Eur: row.p90_eur, + p95Eur: row.p95_eur, + p99Eur: row.p99_eur, + }; +} + +/** + * Coarse position of one value inside its CPV-division cohort. Pure. Three rules keep it from + * over-claiming precision the shared, self-inclusive grid cannot support (review nedda76): + * + * - STRICT `>` for the upper bands, so a value merely EQUAL to an anchor does not qualify. With ties + * (e.g. hundreds of framework contracts at one price) the anchor equals many rows; `>=` labelled + * them all „top 1%". Equal-to-median lands in its own `at-median` band, not „above". + * - DISTINCT anchors: a band is skipped unless its anchor strictly exceeds the next coarser one. In a + * tie-collapsed cohort (p99 = p95 = … = median) none of the fine bands fire and the value reads as + * at/above/below median instead of a false „top 1%". + * - COHORT SIZE: the finer the band, the larger the cohort it needs (BAND_MIN_COHORT) - so the single + * most-expensive contract in a 12-row cohort is never labelled „top 1%". + */ +export function cohortBand(amountEur: number, stats: CpvCohortStats): CohortBand { + const n = stats.pricedContracts; + if (n >= BAND_MIN_COHORT.top1 && stats.p99Eur > stats.p95Eur && amountEur > stats.p99Eur) + return 'top1'; + if (n >= BAND_MIN_COHORT.top5 && stats.p95Eur > stats.p90Eur && amountEur > stats.p95Eur) + return 'top5'; + if (n >= BAND_MIN_COHORT.top10 && stats.p90Eur > stats.p75Eur && amountEur > stats.p90Eur) + return 'top10'; + if (n >= BAND_MIN_COHORT.top25 && stats.p75Eur > stats.medianEur && amountEur > stats.p75Eur) + return 'top25'; + if (amountEur > stats.medianEur) return 'above-median'; + if (amountEur < stats.medianEur) { + // Symmetric lower band, gated the same way (cohort size + distinct anchor + strict <), so the + // single cheapest contract in a small/collapsed cohort is not falsely labelled „bottom 25%". + if (n >= BAND_MIN_COHORT.top25 && stats.medianEur > stats.p25Eur && amountEur < stats.p25Eur) + return 'bottom25'; + return 'below-median'; + } + return 'at-median'; +} + +/** + * The benchmark for one contract from fields getContract already has (no second read), or null when + * no honest comparison exists: no clean value (value_flag <> 'ok' → the shown amount is an estimate, + * not a price), no CPV division, no stats row, or a cohort below MIN_COHORT. The gate matches the + * rollup's WHERE exactly, so a shown contract is always a member of the cohort it is compared against + * (it is NOT leave-one-out — the UI states this). Pure except for the single stats lookup. + */ +export function contractCohort( + amountEur: number | null, + valueFlag: string, + division: string, + stats: CpvCohortStats | null, +): ContractCohortBenchmark | null { + if (amountEur == null || amountEur <= 0 || valueFlag !== 'ok' || !division) return null; + if (!stats || stats.pricedContracts < MIN_COHORT) return null; + return { amountEur, band: cohortBand(amountEur, stats), stats }; +} diff --git a/packages/db/src/queries/details.test.ts b/packages/db/src/queries/details.test.ts index ffe35cffe..9371762fa 100644 --- a/packages/db/src/queries/details.test.ts +++ b/packages/db/src/queries/details.test.ts @@ -56,6 +56,7 @@ function fakeDb( contractRow: typeof baseContractRow, lotRows: unknown[], amendmentRows: unknown[] = [], + cohortStatsRow: unknown = null, ): D1Database { return { prepare(sql: string) { @@ -68,6 +69,8 @@ function fakeDb( async first() { if (sql.includes('WHERE c.id = ?')) return contractRow as T; if (sql.includes('authority_totals') || sql.includes('company_totals')) return null as T; + // The „Подобни договори" cohort lookup — null unless the test supplies a stats row. + if (sql.includes('cpv_division_stats')) return cohortStatsRow as T; throw new Error(`unexpected first query: ${sql}`); }, async all() { @@ -195,6 +198,58 @@ describe('getContract', () => { } }); + // Exercises the real cohort path end-to-end (baseContractRow is clean-value, CPV '72', amount 5000). + // Guards the argument order into contractCohort: swapping value_flag ↔ division would make it return + // null and this would fail. + it('populates the cohort from a real cpv_division_stats row', async () => { + const statsRow = { + division: '72', + priced_contracts: 200, + p25_eur: 1000, + median_eur: 4000, + p75_eur: 10_000, + p90_eur: 40_000, + p95_eur: 90_000, + p99_eur: 400_000, + }; + const detail = await getContract(fakeDb(baseContractRow, [], [], statsRow), 'c:1'); + + expect(detail?.cohort).not.toBeNull(); + expect(detail?.cohort?.amountEur).toBe(5000); // the contract's own amount_eur, not a stats field + expect(detail?.cohort?.stats.division).toBe('72'); + expect(detail?.cohort?.stats.pricedContracts).toBe(200); + expect(detail?.cohort?.band).toBe('above-median'); // 5000 > median 4000, < p75 10000 + }); + + // The read is gated on a clean value: a suspect contract must NOT even query cpv_division_stats. + it('skips the cohort read (and returns no cohort) for a non-clean value', async () => { + let cohortQueried = false; + const db = { + prepare(sql: string) { + if (sql.includes('cpv_division_stats')) cohortQueried = true; + const statement = { + bind() { + return statement; + }, + async first() { + if (sql.includes('WHERE c.id = ?')) + return { ...baseContractRow, value_flag: 'value_suspect' } as T; + return null as T; + }, + async all() { + return { results: [] as T[] }; + }, + }; + return statement; + }, + } as unknown as D1Database; + + const detail = await getContract(db, 'c:1'); + + expect(detail?.cohort).toBeNull(); + expect(cohortQueried).toBe(false); + }); + it('recomputes delta from before/after (ignoring a disagreeing source delta) and trims text', async () => { const detail = await getContract( fakeDb( diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index eb7fe5d31..7a211d8d2 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -20,6 +20,7 @@ import type { } from '@sigma/api-contract'; import { CPV_SECTORS, PROCEDURE_GROUPS, procedureGroup } from '@sigma/config'; import { cleanName, entityName, parseConsortiumMembers } from '@sigma/shared'; +import { contractCohort, getCpvCohortStats } from './cohort'; import { listContracts } from './contracts'; import { authoritySlug, companySlug, contractSlug } from './identity'; import { typeLabel } from './rows'; @@ -491,7 +492,14 @@ export async function getContract( .first(); if (!r) return null; - const [authTotals, compTotals, lotRows, amendmentRows] = await Promise.all([ + // The cohort benchmark only exists for a clean, comparable value (the same gate contractCohort + // applies). Test that gate HERE too, from the row we already read, so a suspect/valueless contract + // skips the cpv_division_stats PK read entirely instead of paying for it and discarding the result + // (the PR's own „one extra rollup read, not a second scan" goal). The division is known + // synchronously, so when it IS read, it loads in parallel with the other detail reads. + const hasCleanValue = r.value_flag === 'ok' && r.amount_eur != null && r.amount_eur > 0; + const cohortDivision = hasCleanValue && r.cpv_code ? r.cpv_code.slice(0, 2) : ''; + const [authTotals, compTotals, lotRows, cohortStats, amendmentRows] = await Promise.all([ db .prepare(`SELECT spent_eur, contracts FROM authority_totals WHERE authority_id = ?`) .bind(r.authority_id) @@ -525,6 +533,7 @@ export async function getContract( bidder_kind: 'company' | 'consortium' | null; bidder_id: string | null; }>(), + cohortDivision ? getCpvCohortStats(db, cohortDivision) : Promise.resolve(null), db.prepare(AMENDMENTS_SQL).bind(r.unp, r.contract_number).all(), ]); @@ -705,6 +714,7 @@ export async function getContract( bidder, lots, subcontractor, + cohort: contractCohort(r.amount_eur, r.value_flag, cohortDivision, cohortStats), amendments, }; diff --git a/packages/db/src/queries/index.ts b/packages/db/src/queries/index.ts index 2ed922e7b..aab63c663 100644 --- a/packages/db/src/queries/index.ts +++ b/packages/db/src/queries/index.ts @@ -15,6 +15,7 @@ export * from './flows'; export * from './network'; export * from './trend'; export * from './regions'; +export * from './cohort'; export * from './competition'; export * from './search'; export * from './details'; diff --git a/scripts/compare-served-sqlite.mjs b/scripts/compare-served-sqlite.mjs index 4589bf7d9..bc0f6a6fd 100644 --- a/scripts/compare-served-sqlite.mjs +++ b/scripts/compare-served-sqlite.mjs @@ -16,6 +16,7 @@ const TABLES = [ 'home_totals', 'sector_totals', 'facet_counts', + 'cpv_division_stats', 'flow_pairs', 'search_index', 'data_freshness', diff --git a/scripts/precompute.sql b/scripts/precompute.sql index 914b3aa78..0eafd0de2 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -146,6 +146,36 @@ INSERT INTO facet_counts (facet, key, contracts, value_eur) SELECT 'eu', CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END, COUNT(*), COALESCE(SUM(c.amount_eur), 0) FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; +-- ── 4c) cpv_division_stats (value percentiles per CPV division - „Подобни договори" benchmark) ── +-- Nearest-rank percentiles (k = ceil(q*n), emulated as CAST(n*q + 0.9999999 AS INTEGER) because +-- SQLite lacks ceil()) over the clean-value cohort: value_flag = 'ok', amount_eur > 0, known CPV. +-- The contract page reads ONE row here instead of scanning its whole division per view (D1 meters +-- rows read). The cohort includes the candidate itself - the shown band is display context and is +-- deliberately coarse, unlike scripts/anomaly-report.mjs which needs leave-one-out p95 for flagging. +CREATE TABLE IF NOT EXISTS cpv_division_stats ( + division TEXT PRIMARY KEY, priced_contracts INTEGER NOT NULL, + p25_eur REAL NOT NULL, median_eur REAL NOT NULL, p75_eur REAL NOT NULL, + p90_eur REAL NOT NULL, p95_eur REAL NOT NULL, p99_eur REAL NOT NULL +); +DELETE FROM cpv_division_stats; +INSERT INTO cpv_division_stats (division, priced_contracts, p25_eur, median_eur, p75_eur, p90_eur, p95_eur, p99_eur) +SELECT division, MAX(cnt), + MAX(CASE WHEN rn = CAST(cnt * 0.25 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.50 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.75 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.90 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.95 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.99 + 0.9999999 AS INTEGER) THEN amount_eur END) +FROM ( + SELECT substr(t.cpv_code, 1, 2) AS division, c.amount_eur, + ROW_NUMBER() OVER (PARTITION BY substr(t.cpv_code, 1, 2) ORDER BY c.amount_eur, c.id) AS rn, + COUNT(*) OVER (PARTITION BY substr(t.cpv_code, 1, 2)) AS cnt + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE c.amount_eur IS NOT NULL AND c.amount_eur > 0 AND c.value_flag = 'ok' + AND COALESCE(t.cpv_code, '') <> '' +) +GROUP BY division; + -- ── 5) flow_pairs (per authority → bidder) ────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS flow_pairs ( authority_id TEXT NOT NULL REFERENCES authorities(id), bidder_id TEXT NOT NULL REFERENCES bidders(id), diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index ff333431f..3324fdfe1 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1888,6 +1888,37 @@ INSERT INTO facet_counts (facet, key, contracts, value_eur) SELECT 'eu', CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END, COUNT(*), COALESCE(SUM(c.amount_eur), 0) FROM contracts c GROUP BY CASE WHEN c.eu_funded = 1 THEN '1' ELSE '0' END; +-- @refresh-batch cohort-stats +-- cpv_division_stats: same full rebuild as scripts/precompute.sql §4c. Runs in its OWN batch (not in +-- `globals` with data_freshness/home_totals/sector_totals/facet_counts) because it is the heaviest +-- statement of the refresh - a full ~150k-row PARTITION BY window sort + rank over the whole corpus. +-- Bundling it into `globals` risked blowing that single D1 request's CPU budget and failing the entire +-- step; its own batch isolates that cost, matching how the other heavy rollups get dedicated batches +-- (review nedda76). IF NOT EXISTS bootstraps a served DB that predates the migration. +CREATE TABLE IF NOT EXISTS cpv_division_stats ( + division TEXT PRIMARY KEY, priced_contracts INTEGER NOT NULL, + p25_eur REAL NOT NULL, median_eur REAL NOT NULL, p75_eur REAL NOT NULL, + p90_eur REAL NOT NULL, p95_eur REAL NOT NULL, p99_eur REAL NOT NULL +); +DELETE FROM cpv_division_stats; +INSERT INTO cpv_division_stats (division, priced_contracts, p25_eur, median_eur, p75_eur, p90_eur, p95_eur, p99_eur) +SELECT division, MAX(cnt), + MAX(CASE WHEN rn = CAST(cnt * 0.25 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.50 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.75 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.90 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.95 + 0.9999999 AS INTEGER) THEN amount_eur END), + MAX(CASE WHEN rn = CAST(cnt * 0.99 + 0.9999999 AS INTEGER) THEN amount_eur END) +FROM ( + SELECT substr(t.cpv_code, 1, 2) AS division, c.amount_eur, + ROW_NUMBER() OVER (PARTITION BY substr(t.cpv_code, 1, 2) ORDER BY c.amount_eur, c.id) AS rn, + COUNT(*) OVER (PARTITION BY substr(t.cpv_code, 1, 2)) AS cnt + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE c.amount_eur IS NOT NULL AND c.amount_eur > 0 AND c.value_flag = 'ok' + AND COALESCE(t.cpv_code, '') <> '' +) +GROUP BY division; + -- @refresh-batch cleanup DROP TABLE IF EXISTS refresh_joint_tender_leads; DROP TABLE IF EXISTS refresh_unp_prefix_authorities;