diff --git a/apps/web/app/components/SubjectRiskIndicator.tsx b/apps/web/app/components/SubjectRiskIndicator.tsx new file mode 100644 index 000000000..2c399b440 --- /dev/null +++ b/apps/web/app/components/SubjectRiskIndicator.tsx @@ -0,0 +1,57 @@ +import { Link } from 'react-router'; +import { count, plural, pct } from '@sigma/shared'; +import { Callout } from './ui'; +import type { RiskBandKey, RiskComponentKey, SubjectRiskView } from '../lib/subjectRisk'; + +const BAND_LABEL: Record = { + few: 'Малко индикатори', + some: 'Единични индикатори', + many: 'Множество индикатори', + most: 'Много индикатори — заслужава преглед', +}; + +const COMPONENT_LABEL: Record = { + single_offer: 'Една оферта', + high_markup: 'Високо оскъпяване', +}; + +// Each component's „виж договорите" links to exactly the contracts it counts, so every number on the +// page is traceable (the drill-down control, M7). The value is the same predicate the rollup +// materializes: bids=1 ⇒ c.bids_received = 1, markup=high ⇒ c.is_high_markup = 1 (@sigma/db filters). +const COMPONENT_FILTER: Record = { + single_offer: 'bids=1', + high_markup: 'markup=high', +}; + +// Subject-level risk. Rendered ONLY when buildSubjectRisk returned a view (natural persons and thin +// samples are already suppressed upstream). The framing, the band, the counts and the drill-down are one +// atomic block (M8) — the disclaimer never renders apart from the number, and the caller keeps this out +// of /OG so it can't become a search snippet. +export function SubjectRiskIndicator({ + risk, + contractsBase, +}: { + risk: SubjectRiskView; + contractsBase: string; // e.g. '/contracts?bidder=103267194' (companySlug: EIK w/o prefix) or '?authority=' — already carries a query string +}) { + return ( + +

+ Обобщава колко от договорите на субекта имат рискови признаци. Неутрален индикатор — не + оценява процедурите и не маркира субекта като нарушител. Изводите прави потребителят.{' '} + Методология. +

+

{BAND_LABEL[risk.band]}

+
    + {risk.components.map((c) => ( +
  • + {COMPONENT_LABEL[c.key]}: {count(c.k)} от {count(c.n)}{' '} + {plural(c.n, 'договор', 'договора')} + {c.valueShare != null ? <> · {pct(c.valueShare)} от стойността : null} ·{' '} + виж договорите +
    • + ))} +
+ + ); +} diff --git a/apps/web/app/components/ui.tsx b/apps/web/app/components/ui.tsx index dd1c18d74..d7ed6d254 100644 --- a/apps/web/app/components/ui.tsx +++ b/apps/web/app/components/ui.tsx @@ -83,7 +83,7 @@ export function Callout({ children, }: { title?: ReactNode; - variant?: 'warning'; + variant?: 'warning' | 'neutral'; children: ReactNode; }) { return ( diff --git a/apps/web/app/lib/csv-export.test.ts b/apps/web/app/lib/csv-export.test.ts index 105c47784..f9b27f844 100644 --- a/apps/web/app/lib/csv-export.test.ts +++ b/apps/web/app/lib/csv-export.test.ts @@ -307,6 +307,7 @@ describe('isUnfilteredCsvExport', () => { ['bidder', { bidder: 'acme' }], ['q', { q: 'rail' }], ['bids', { bids: 'one' }], + ['markup', { markup: 'high' }], ['companies.kinds', { kinds: ['company'] }], ['companies.countBucket', { countBucket: '2-5' }], ['authorities.types', { types: ['municipality'] }], @@ -335,6 +336,7 @@ describe('isUnfilteredCsvExport', () => { bidder: 'acme', q: 'rail', bids: 'one', + markup: 'high', types: ['municipality'], kinds: ['company'], countBucket: '2-5', diff --git a/apps/web/app/lib/csv-export.ts b/apps/web/app/lib/csv-export.ts index 4756efdaa..c7f2d4ad0 100644 --- a/apps/web/app/lib/csv-export.ts +++ b/apps/web/app/lib/csv-export.ts @@ -9,7 +9,16 @@ const ARRAY_FILTERS = ['years', 'sectors', 'procedureGroups', 'kinds', 'types'] // was misclassified as unfiltered and served from / written to the shared unfiltered cache object — // a cache-poisoning variant of #56/#122 on top of the wrong-data bug (#138). hasScalarFilter treats // 'one' as set and null as absent, so it slots in cleanly. Other routes simply never carry the key. -const SCALAR_FILTERS = ['valueBucket', 'eu', 'authority', 'bidder', 'countBucket', 'bids'] as const; +// `markup` ('high' | null) is response-affecting exactly like `bids` — same #138 cache-poisoning class. +const SCALAR_FILTERS = [ + 'valueBucket', + 'eu', + 'authority', + 'bidder', + 'countBucket', + 'bids', + 'markup', +] as const; const FILENAMES = { contracts: 'sigma-contracts.csv', companies: 'sigma-companies.csv', diff --git a/apps/web/app/lib/filters.ts b/apps/web/app/lib/filters.ts index 6e5d621bb..33a5288e5 100644 --- a/apps/web/app/lib/filters.ts +++ b/apps/web/app/lib/filters.ts @@ -51,6 +51,7 @@ export function contractListFilters(sp: URLSearchParams) { bidder: sp.get('bidder'), q: sp.get('q'), bids: (sp.get('bids') === '1' ? 'one' : null) as 'one' | null, + markup: (sp.get('markup') === 'high' ? 'high' : null) as 'high' | null, }; } @@ -190,6 +191,7 @@ export const PARAM_ORDER = [ 'funding', 'eu', 'bids', // /contracts single-bid filter + 'markup', // /contracts high-markup filter (risk drill-down) 'value', 'authority', 'bidder', diff --git a/apps/web/app/lib/query-params.ts b/apps/web/app/lib/query-params.ts index e7b603a30..c2587795b 100644 --- a/apps/web/app/lib/query-params.ts +++ b/apps/web/app/lib/query-params.ts @@ -6,6 +6,7 @@ export const CANONICAL_QUERY_PARAMS = new Set([ 'bidder', 'bids', // single-bid filter — changes the result set + totals 'center', + 'markup', // /contracts: c.is_high_markup = 1 — changes the result set + totals (must stay keyed so the risk drill-down link survives withParams) 'count', 'cursor', 'eu', diff --git a/apps/web/app/lib/riskLogic.test.ts b/apps/web/app/lib/riskLogic.test.ts index 11d512927..0e9adb1c7 100644 --- a/apps/web/app/lib/riskLogic.test.ts +++ b/apps/web/app/lib/riskLogic.test.ts @@ -1,77 +1,85 @@ import { describe, it, expect } from 'vitest'; -import { evaluateRiskIndicators } from './riskLogic'; +import { evaluateRiskIndicators, type RiskFlagInput } from './riskLogic'; -function buildContract(overrides: any = {}): any { +// evaluateRiskIndicators reads the materialized flags (isSingleOffer/isHighMarkup) — the same columns the +// subject-risk rollups aggregate (#229) — and deltaPct only for the displayed %. A ContractDetail +// satisfies RiskFlagInput structurally. +function buildContract(overrides: Partial = {}): RiskFlagInput { return { - bidsReceived: 2, - bidsRejected: 0, + isSingleOffer: false, + isHighMarkup: false, euFunded: false, dateSuspect: false, - value: { - deltaPct: 0.1, - suspect: false, - }, + value: { deltaPct: 0.1, suspect: false }, ...overrides, }; } describe('evaluateRiskIndicators', () => { - it('returns empty when no risks are present', () => { - const flags = evaluateRiskIndicators(buildContract()); - expect(flags).toHaveLength(0); + it('returns empty when no flags are set', () => { + expect(evaluateRiskIndicators(buildContract())).toEqual([]); }); - describe('Competition heuristics', () => { - it('triggers NO_COMPETITION when exactly 1 bid is admitted (non-EU)', () => { - const contract = buildContract({ bidsReceived: 3, bidsRejected: 2, euFunded: false }); - const flags = evaluateRiskIndicators(contract); - expect(flags).toEqual([{ type: 'no_competition' }]); + describe('competition', () => { + it('flags no_competition when single-offer and not EU funded', () => { + expect( + evaluateRiskIndicators(buildContract({ isSingleOffer: true, euFunded: false })), + ).toEqual([{ type: 'no_competition' }]); }); - it('triggers EU_NO_COMPETITION when exactly 1 bid is admitted and EU funded', () => { - const contract = buildContract({ bidsReceived: 1, bidsRejected: 0, euFunded: true }); - const flags = evaluateRiskIndicators(contract); - expect(flags).toEqual([{ type: 'eu_no_competition' }]); + it('flags eu_no_competition when single-offer and EU funded', () => { + expect( + evaluateRiskIndicators(buildContract({ isSingleOffer: true, euFunded: true })), + ).toEqual([{ type: 'eu_no_competition' }]); }); - it('does not trigger competition flags when > 1 bid is admitted', () => { - const contract = buildContract({ bidsReceived: 2, bidsRejected: 0 }); - const flags = evaluateRiskIndicators(contract); - expect(flags).not.toContainEqual({ type: 'no_competition' }); - expect(flags).not.toContainEqual({ type: 'eu_no_competition' }); + it('does not flag competition when not single-offer', () => { + expect(evaluateRiskIndicators(buildContract({ isSingleOffer: false }))).toEqual([]); + }); + + it('does not flag when the single-offer flag is null (unknown bid count)', () => { + // The flag is the sole input — unified on bids_received = 1; there is no bid-count arithmetic here. + expect(evaluateRiskIndicators(buildContract({ isSingleOffer: null }))).toEqual([]); }); }); - describe('Markup heuristics', () => { - it('triggers HIGH_MARKUP when deltaPct > 20%', () => { - const contract = buildContract({ value: { deltaPct: 0.21, suspect: false } }); - const flags = evaluateRiskIndicators(contract); - expect(flags).toContainEqual({ type: 'high_markup', deltaPct: 0.21 }); + describe('markup', () => { + it('flags high_markup when the flag is set, carrying deltaPct for display', () => { + expect( + evaluateRiskIndicators( + buildContract({ isHighMarkup: true, value: { deltaPct: 0.21, suspect: false } }), + ), + ).toEqual([{ type: 'high_markup', deltaPct: 0.21 }]); + }); + + it('does not flag high_markup when the flag is false, even with a high deltaPct', () => { + expect( + evaluateRiskIndicators( + buildContract({ isHighMarkup: false, value: { deltaPct: 0.5, suspect: false } }), + ), + ).toEqual([]); }); - it('does not trigger HIGH_MARKUP when deltaPct is exactly 20% or less', () => { - const contract1 = buildContract({ value: { deltaPct: 0.2, suspect: false } }); - const contract2 = buildContract({ value: { deltaPct: 0.19, suspect: false } }); - expect(evaluateRiskIndicators(contract1)).not.toContainEqual( - expect.objectContaining({ type: 'high_markup' }), - ); - expect(evaluateRiskIndicators(contract2)).not.toContainEqual( - expect.objectContaining({ type: 'high_markup' }), - ); + it('does not flag high_markup when the flag is set but deltaPct is null (no NaN%)', () => { + expect( + evaluateRiskIndicators( + buildContract({ isHighMarkup: true, value: { deltaPct: null, suspect: false } }), + ), + ).toEqual([]); }); }); - describe('Anomaly heuristics', () => { - it('triggers ANOMALIES when date is suspect', () => { - const contract = buildContract({ dateSuspect: true }); - const flags = evaluateRiskIndicators(contract); - expect(flags).toContainEqual({ type: 'anomalies' }); + describe('anomalies', () => { + it('flags anomalies when the date is suspect', () => { + expect(evaluateRiskIndicators(buildContract({ dateSuspect: true }))).toEqual([ + { type: 'anomalies' }, + ]); }); - it('triggers ANOMALIES when value is suspect', () => { - const contract = buildContract({ value: { deltaPct: 0, suspect: true } }); - const flags = evaluateRiskIndicators(contract); - expect(flags).toContainEqual({ type: 'anomalies' }); + it('flags anomalies when the value is suspect', () => { + expect( + evaluateRiskIndicators(buildContract({ value: { deltaPct: 0, suspect: true } })), + ).toEqual([{ type: 'anomalies' }]); }); }); }); diff --git a/apps/web/app/lib/riskLogic.ts b/apps/web/app/lib/riskLogic.ts index db6c0b6ec..be7932df8 100644 --- a/apps/web/app/lib/riskLogic.ts +++ b/apps/web/app/lib/riskLogic.ts @@ -7,25 +7,28 @@ export interface RiskIndicatorResult { deltaPct?: number; } -export function evaluateRiskIndicators(contract: ContractDetail): RiskIndicatorResult[] { +/** The fields evaluateRiskIndicators actually reads — a ContractDetail satisfies this structurally. + * isSingleOffer/isHighMarkup are the materialized flags (scripts/precompute.sql), so the per-contract + * display and the subject-risk rollups share ONE definition, unified on `bids_received = 1`. */ +export type RiskFlagInput = Pick< + ContractDetail, + 'isSingleOffer' | 'isHighMarkup' | 'euFunded' | 'dateSuspect' +> & { value: Pick }; + +export function evaluateRiskIndicators(contract: RiskFlagInput): RiskIndicatorResult[] { const flags: RiskIndicatorResult[] = []; - const admitted = - contract.bidsReceived != null ? contract.bidsReceived - (contract.bidsRejected || 0) : null; - - if (admitted === 1) { - if (contract.euFunded) { - flags.push({ type: 'eu_no_competition' }); - } else { - flags.push({ type: 'no_competition' }); - } + if (contract.isSingleOffer) { + flags.push({ type: contract.euFunded ? 'eu_no_competition' : 'no_competition' }); } - if (contract.value?.deltaPct != null && contract.value.deltaPct > 0.2) { + // isHighMarkup is the materialized flag; deltaPct is still read for the displayed %. It is NULL on the + // suspect rows where the flag is also null, so the `!= null` guard stops a stale flag rendering `NaN%`. + if (contract.isHighMarkup && contract.value.deltaPct != null) { flags.push({ type: 'high_markup', deltaPct: contract.value.deltaPct }); } - if (contract.dateSuspect || contract.value?.suspect) { + if (contract.dateSuspect || contract.value.suspect) { flags.push({ type: 'anomalies' }); } diff --git a/apps/web/app/lib/subjectRisk.test.ts b/apps/web/app/lib/subjectRisk.test.ts new file mode 100644 index 000000000..bf1cbde15 --- /dev/null +++ b/apps/web/app/lib/subjectRisk.test.ts @@ -0,0 +1,88 @@ +import { describe, it, expect } from 'vitest'; +import type { SubjectRiskAggregate } from '@sigma/api-contract'; +import { buildSubjectRisk } from './subjectRisk'; + +function agg(o: Partial = {}): SubjectRiskAggregate { + return { + singleOfferK: null, + singleOfferN: null, + singleOfferValueShare: null, + highMarkupK: null, + highMarkupN: null, + highMarkupValueShare: null, + ...o, + }; +} + +describe('buildSubjectRisk', () => { + it('suppresses everything for a natural-person profile (M9)', () => { + expect( + buildSubjectRisk(agg({ singleOfferK: 5, singleOfferN: 5 }), { isNaturalPerson: true }), + ).toBeNull(); + }); + + it('returns null when there is no aggregate row', () => { + expect(buildSubjectRisk(null, { isNaturalPerson: false })).toBeNull(); + }); + + it('suppresses when no component has enough assessable contracts (min-N, M3)', () => { + expect( + buildSubjectRisk(agg({ singleOfferK: 4, singleOfferN: 4, highMarkupK: 2, highMarkupN: 2 }), { + isNaturalPerson: false, + }), + ).toBeNull(); + }); + + it('reports a component exactly at the min-N boundary (n = 5)', () => { + expect( + buildSubjectRisk(agg({ singleOfferK: 1, singleOfferN: 5 }), { isNaturalPerson: false }), + ).toEqual({ + composite: 0.2, + band: 'some', + components: [{ key: 'single_offer', k: 1, n: 5, countShare: 0.2, valueShare: null }], + }); + }); + + it('bands a zero composite as „few"', () => { + expect( + buildSubjectRisk(agg({ singleOfferK: 0, singleOfferN: 5 }), { isNaturalPerson: false })?.band, + ).toBe('few'); + }); + + it('bands a full composite as „most"', () => { + expect( + buildSubjectRisk(agg({ singleOfferK: 5, singleOfferN: 5 }), { isNaturalPerson: false })?.band, + ).toBe('most'); + }); + + it('averages reportable components for the composite (count-weighted)', () => { + // single-offer 5/5 = 1.0, high-markup 0/5 = 0.0 → composite 0.5 → „many" (< 0.55). + const view = buildSubjectRisk( + agg({ singleOfferK: 5, singleOfferN: 5, highMarkupK: 0, highMarkupN: 5 }), + { isNaturalPerson: false }, + ); + expect(view?.composite).toBe(0.5); + expect(view?.band).toBe('many'); + expect(view?.components).toHaveLength(2); + }); + + it('drops a thin component from the composite but keeps the reportable one', () => { + // single-offer n=5 reportable; high-markup n=3 dropped → composite is single-offer only. + const view = buildSubjectRisk( + agg({ singleOfferK: 3, singleOfferN: 5, highMarkupK: 3, highMarkupN: 3 }), + { isNaturalPerson: false }, + ); + expect(view?.components).toEqual([ + { key: 'single_offer', k: 3, n: 5, countShare: 0.6, valueShare: null }, + ]); + expect(view?.composite).toBe(0.6); + }); + + it('passes the value share through unchanged', () => { + const view = buildSubjectRisk( + agg({ singleOfferK: 3, singleOfferN: 5, singleOfferValueShare: 0.42 }), + { isNaturalPerson: false }, + ); + expect(view?.components[0]?.valueShare).toBe(0.42); + }); +}); diff --git a/apps/web/app/lib/subjectRisk.ts b/apps/web/app/lib/subjectRisk.ts new file mode 100644 index 000000000..6631fef16 --- /dev/null +++ b/apps/web/app/lib/subjectRisk.ts @@ -0,0 +1,69 @@ +import type { SubjectRiskAggregate } from '@sigma/api-contract'; + +// Presentation thresholds — server-side constants, NEVER query params (ADR-0007). Band cutoffs are +// provisional, to be calibrated against the real distribution. Pure logic: keys + numbers only — the +// Bulgarian band/component labels live in the SubjectRiskIndicator component, not here. +export const MIN_ELIGIBLE = 5; // a component needs ≥ this many assessable contracts to be reportable (M3) + +export type RiskBandKey = 'few' | 'some' | 'many' | 'most'; +export type RiskComponentKey = 'single_offer' | 'high_markup'; + +// The band is chosen from the count-weighted composite (robust to one dominant contract); value shares +// are context only. 'most' (Infinity) always matches, so bandFor never falls through. +const BAND_CUTOFFS: readonly { key: RiskBandKey; below: number }[] = [ + { key: 'few', below: 0.1 }, + { key: 'some', below: 0.3 }, + { key: 'many', below: 0.55 }, + { key: 'most', below: Infinity }, +]; + +function bandFor(composite: number): RiskBandKey { + return BAND_CUTOFFS.find((b) => composite < b.below)?.key ?? 'most'; +} + +export interface SubjectRiskComponent { + key: RiskComponentKey; + k: number; // flagged contracts + n: number; // eligible contracts (the „K от N" denominator) + countShare: number; // k / n ∈ [0,1] + valueShare: number | null; // ∈ [0,1], or null when no positive eligible value +} + +export interface SubjectRiskView { + composite: number; // mean of the reportable components' count shares ∈ [0,1] + band: RiskBandKey; + components: SubjectRiskComponent[]; // reportable only (n ≥ MIN_ELIGIBLE) +} + +function toReportable( + key: RiskComponentKey, + k: number | null, + n: number | null, + valueShare: number | null, +): SubjectRiskComponent | null { + if (n == null || n < MIN_ELIGIBLE) return null; + // null k (flag unmaterialized) → 0 flagged: missing data under-reports risk, never invents it. + const flagged = k ?? 0; + return { key, k: flagged, n, countShare: flagged / n, valueShare }; +} + +/** Display-ready subject risk, or null when it must be suppressed: a natural-person profile (M9) or no + * component with enough assessable contracts to be reportable (M3 — then no band and no score render). */ +export function buildSubjectRisk( + agg: SubjectRiskAggregate | null, + opts: { isNaturalPerson: boolean }, +): SubjectRiskView | null { + if (opts.isNaturalPerson || agg == null) return null; + + const components = [ + toReportable('single_offer', agg.singleOfferK, agg.singleOfferN, agg.singleOfferValueShare), + toReportable('high_markup', agg.highMarkupK, agg.highMarkupN, agg.highMarkupValueShare), + ].filter((c): c is SubjectRiskComponent => c !== null); + + if (components.length === 0) return null; + + // Mean over the REPORTABLE components only — a thin (< MIN_ELIGIBLE) component is dropped, not scored + // as zero (M3 small-sample conservatism). Subjects stand alone (no cross-subject ranking). + const composite = components.reduce((sum, c) => sum + c.countShare, 0) / components.length; + return { composite, band: bandFor(composite), components }; +} diff --git a/apps/web/app/routes/authority.tsx b/apps/web/app/routes/authority.tsx index 54eb87120..853ff960d 100644 --- a/apps/web/app/routes/authority.tsx +++ b/apps/web/app/routes/authority.tsx @@ -19,7 +19,9 @@ import { TrendChart } from '../components/TrendChart'; import { NetworkGraph } from '../components/NetworkGraph'; import { ContractMiniTable } from '../components/ContractMiniTable'; import { EuBenchmarkStat } from '../components/EuBenchmarkStat'; +import { SubjectRiskIndicator } from '../components/SubjectRiskIndicator'; import { ShareBar, Chip, Section } from '../components/ui'; +import { buildSubjectRisk } from '../lib/subjectRisk'; import { publicCache } from '../lib/cache'; import { coverageRange, getCoverageMeta } from '../lib/coverage'; import { networkColumns, networkRows, trendYearColumns } from '../lib/entity-tables'; @@ -77,6 +79,7 @@ export default function Authority({ loaderData }: Route.ComponentProps) { procedure.nonCompetitiveShare, EU_SCOREBOARD.directAward, ); + const risk = buildSubjectRisk(a.risk, { isNaturalPerson: false }); const range = coverageRange(loaderData.coverage.coverageEndYear); const topSectors = a.sectors .slice(0, 3) @@ -136,6 +139,12 @@ export default function Authority({ loaderData }: Route.ComponentProps) { ]} /> + {risk ? ( +
+ +
+ ) : null} +
+ {risk ? ( +
+ +
+ ) : null} +

- СИГМА е изцяло само за четене: не въвежда нови данни, не оценява процедурите и не - маркира фирми като рискови. + СИГМА има само информативен характер: не въвежда нови данни и не оценява + процедурите. Показва неутрални индикатори, обобщени от самите договори — те не са + обвинение и не установяват нарушение.

@@ -361,6 +362,22 @@ export default function Methodology({ loaderData }: Route.ComponentProps) {

→ bids_received = 1 +
Обобщен рисков индикатор
+
+

+ Обобщава на ниво субект (възложител или изпълнител) колко от договорите му имат{' '} + рискови признаци — една оферта и високо оскъпяване — по подхода + CRI на Government Transparency Institute; изчислява се и по брой, и по стойност. +

+

+ Показва се само при поне 5 договора с достатъчно данни за + признака; за физически лица не се показва. Категорията („Малко" → „Много + индикатори") е неутрална и описва индикаторите, не субекта; праговете са + временни и подлежат на калибриране. „Високо оскъпяване" не включва анексите с + подозрителни стойности, затова е консервативна долна оценка. +

+ → scripts/precompute.sql · bids_received = 1 +
Концентрация на доставчици (HHI)

diff --git a/apps/web/app/styles/components.css b/apps/web/app/styles/components.css index 6acaffcff..f75915acb 100644 --- a/apps/web/app/styles/components.css +++ b/apps/web/app/styles/components.css @@ -175,6 +175,35 @@ a.flag:hover { background: var(--accent-bg); color: var(--ink); } +/* #229 subject-risk framing — subdued and non-alarming (never the red warning accent). */ +.callout.neutral { + border-left-color: var(--ink-soft); + background: var(--surface); +} +.subject-risk-band { + margin: var(--s-3) 0 var(--s-2); + font: 400 18px/1.25 var(--font-serif); + color: var(--ink); +} +.subject-risk-list { + margin: 0; + padding: 0; + list-style: none; +} +.subject-risk-list li { + padding: 3px 0; +} +/* Bigger touch target for „виж договорите" on mobile (WCAG 2.5.8 ≥ 24px): a little row spacing plus + link padding, sized so the two components' links never overlap (a mis-tap would swap the filter). */ +@media (max-width: 640px) { + .subject-risk-list li { + padding: 6px 0; + } + .subject-risk-list a { + display: inline-block; + padding: 4px 0; + } +} .callout h2, .callout h3 { font: 400 18px/1.25 var(--font-serif); diff --git a/docs/README.md b/docs/README.md index 568550ad6..758a7b532 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ - [`etl-pipeline-state.md`](etl-pipeline-state.md) — анализ на текущото състояние на ETL pipeline-а. - [`etl-architecture.md`](etl-architecture.md) — целевата ETL архитектура (RFC): предложение за състоянието и реда на изпълнение. - [`v1-implementation-plan.md`](v1-implementation-plan.md) — precompute слоят и пагинацията (защо rollup-и и keyset вместо per-request GROUP BY / OFFSET). +- [`implementation-plans/229-subject-risk-composite.md`](implementation-plans/229-subject-risk-composite.md) — планът за #229 (композитен рисков индикатор на ниво субект): компоненти, прагове, тегла и анти-обвинителната рамка (M1–M9). - [`integrity-gate.md`](integrity-gate.md) — reconciliation gate-ът: hard asserts върху тоталите при import/CI. - [`anomaly-report.md`](anomaly-report.md) — cross-row аномалии при опресняване: какво `value_flag` не хваща на ниво отделен договор. - [`deploy.md`](deploy.md) — деплой към Cloudflare: двата Worker-а (`sigma`, `sigma-etl`) и споделеният D1 per environment. diff --git a/docs/adr/0002-d1-as-datastore.md b/docs/adr/0002-d1-as-datastore.md index 1678c3089..62db3d28a 100644 --- a/docs/adr/0002-d1-as-datastore.md +++ b/docs/adr/0002-d1-as-datastore.md @@ -16,9 +16,11 @@ Ползваме **Cloudflare D1** (SQLite на edge) като единствено обслужвано хранилище — **един D1 на среда**, споделян от двата Worker-а: `sigma` чете, `sigma-etl` пише (виж [`deploy.md`](../deploy.md)). Схемата е **консолидиран baseline** ([`0000_init.sql`](../../packages/db/migrations/0000_init.sql)) плюс -тънки добавъчни миграции (напр. `0001_flow_pairs_bidder_index.sql`), а не верига от самото начало — -v1 е pre-production и всеки импорт стартира от свежа база; пълна верига инкрементални миграции се -въвежда чак когато има деплойнати данни, които не може да се загубят. +тънки добавъчни миграции (напр. `0001_flow_pairs_bidder_index.sql`), а не верига от самото начало. +**Work** базата се пресъздава при всеки импорт от `0000_init`, но **обслужваната D1 е персистентна** — +`wrangler d1 migrations apply` е filename-tracked, тъй че приложен файл (вкл. `0000_init`) е замразен +там. Затова нов schema обект се добавя с нова номерирана миграция (`ALTER TABLE …`), а не чрез +редакция на приложения `0000_init` — редакция стига до work базата и локалното CI, но никога до прод. ## Последствия diff --git a/docs/adr/0007-subject-risk-composite.md b/docs/adr/0007-subject-risk-composite.md new file mode 100644 index 000000000..0b89d0453 --- /dev/null +++ b/docs/adr/0007-subject-risk-composite.md @@ -0,0 +1,52 @@ +# ADR-0007 — Композитен рисков индикатор на ниво субект (възложител/изпълнител) + +- **Статус:** Предложено +- **Дата:** 2026-07-14 +- **Обхват:** #229 — `scripts/precompute.sql` + `scripts/refresh-slice.sql` (агрегати), + `apps/web/app/lib/riskLogic.ts` и профилите на компания/възложител. + +## Контекст + +`riskLogic.ts` дава елементарни флагове на ниво **договор** (една оферта, високо оскъпяване), но +липсва агрегат на ниво **субект**. CRI методологията (Government Transparency Institute) агрегира +сигналите от отделните поръчки към организация — възприемаме я като външна, рецензирана +**методологична основа**. D1 таксува по прочетени редове, затова агрегатите се смятат **предварително +(precompute), а не на всяка заявка**. + +Две несъвместими дефиниции за „една оферта" съществуват: `bids_received = 1` (в `competition.ts`, +`describe-schema.ts` — вече показвани числа) и `admitted === 1`, т.е. `bids_received − bids_rejected = 1` +(в `riskLogic.ts`). Двете дават различни числа за един и същ субект. + +Показателят е **чувствителен към клевета**: `methodology.tsx` обещава публично, че СИГМА „не маркира +фирми като рискови". Композитна категория до името на субект **противоречи на това обещание**. + +## Решение + +**Композитният показател е средната стойност на двата компонента — дял „една оферта" и дял „високо +оскъпяване" — с равни тегла.** Всеки компонент се измерва по два начина, поотделно: **по брой договори** +и **по стойност**. Равните тегла са нарочен избор (за **устойчивост** и защитимост); всяко отклонение +би искало обосновка. + +„Една оферта" се **унифицира на `bids_received = 1`** навсякъде — това е вече показваната база; и +per-contract флагът в `riskLogic.ts` се пренасочва към нея. Материализираме канонични булеви колони +`is_single_offer`/`is_high_markup` на `contracts` (единствен източник; NULL = неизвестно). Всеки +компонент дели по **своя допустим знаменател** (една оферта → `bids_received >= 1`; оскъпяване → +ненулеви `signing_value_eur`/`current_value_eur`). + +**Неутрална рамка (задължителна):** етикетите описват _индикаторите_, не субекта — „Малко индикатори" / +„Единични индикатори" / „Множество индикатори" / „Много индикатори — заслужава преглед"; никога +„критичен" / „корупция" / „нередност" срещу именуван субект. **Категорията се определя от композита, +претеглен по брой договори (така един-единствен голям договор не може сам да я вдигне), а версията по +стойност се показва само като допълнителен контекст.** Праговете и минималната извадка (**N ≥ 5** +допустими договора на компонент) са сървърни константи, не query-параметри. Показателят се **скрива за +профили на физически лица** и не влиза в ``/OG. Праговете за категориите са **временни, за +калибриране** спрямо реалното разпределение. + +## Последствия + +- Единна дефиниция за „една оферта" — композитът съвпада с числото на страницата на възложителя. +- **Промяна в поведението** на per-contract флага: договор с 3 оферти и 2 отхвърлени вече **не + задейства** флага „липса на конкуренция" (по-твърд, по-защитим факт). +- „Много оферти, повечето отхвърлени" се **отлага като отделен бъдещ флаг** — не се губи, разделя се. +- Преформулирането на изречението в `methodology.tsx` изисква **одобрение от maintainer** (отделен commit). +- Follow-up: калибриране на праговете; #153 (пряко възлагане) и #41/#210 (CPV кохорта) като нови флагове. diff --git a/docs/adr/README.md b/docs/adr/README.md index 600cc7a6a..7a1f3fa29 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -14,3 +14,4 @@ | [0004](0004-style-src-unsafe-inline.md) | `style-src` запазва `'unsafe-inline'` (CSP) | Прието | | [0005](0005-blue-green-d1-rollback.md) | Blue/green D1 слотове за rollback на refresh | Прието | | [0006](0006-eop-wins-dedup.md) | Dedup на два източника: EOP печели по `contract_number` | Прието | +| [0007](0007-subject-risk-composite.md) | Композитен рисков индикатор на ниво субект (възложител/изпълнител) | Предложено | diff --git a/docs/etl.md b/docs/etl.md index a060a6853..13f589fb6 100644 --- a/docs/etl.md +++ b/docs/etl.md @@ -337,7 +337,9 @@ per-day bucket-ите на `storage.eop.bg` (кеширани в `data/eop/`). `search_index`. `contracts.current_value`/`annex_count` остават (rollup-ът; `amendments` е source историята му). -Репото е pre-production и ползва един свеж стартов schema файл — `packages/db/migrations/0000_init.sql`. +Work базата се пресъздава при всеки импорт от `packages/db/migrations/0000_init.sql`; обслужваната D1 е +персистентна и получава номерираните миграции чрез `wrangler d1 migrations apply`, затова нови schema +обекти минават през нова номерирана миграция, а не през редакция на `0000_init`. Допълненията за `storage.eop` включват контактни полета на възложители и изпълнители, `raw_ocds_lots` (в work схемата) и стойностни полета на `lots`. Обслужваният `tenders` ред носи суровия EOP числов `tenderId` (`eop_tender_id`, миграция `0003_tender_eop_id.sql`), за да може diff --git a/docs/implementation-plans/229-subject-risk-composite.md b/docs/implementation-plans/229-subject-risk-composite.md new file mode 100644 index 000000000..98fca7ebc --- /dev/null +++ b/docs/implementation-plans/229-subject-risk-composite.md @@ -0,0 +1,148 @@ +# Implementation Plan: #229 — Composite subject-level risk indicator (CRI-style) + +- **Status:** Draft — awaiting approval +- **Created:** 2026-07-14 +- **Branch:** `feat/subject-risk-composite` (off `origin/main`) +- **Delivery:** ONE PR, ~6 focused commits (see §7) + +## 1. Executive summary + +Aggregate the two elementary per-contract risk flags that exist today (single-offer, high-markup) +into a **composite risk indicator per subject** (company / authority), computed by contract **count** +and by **value** (separately), and surface it on the profile pages as a **neutral indicator** with a +band, a component breakdown, the underlying counts, and a drill-down to the exact contracts. + +- **Complexity:** Medium-High (touches the ETL pipeline, the daily refresh, the contract-detail read + path, and two profile pages). +- **Risk level:** Medium — defamation-sensitive (a risk label on a named subject on a minister-visible + site). Mitigated by the framing controls in §5. + +## 2. Decisions locked (with the human who owns each) + +| # | Decision | Choice | Owner | +|---|---|---|---| +| A | The `methodology.tsx` promise „не маркира фирми като рискови" contradicts a risk band | **Keep the promise; reframe the feature as a neutral aggregate indicator** and reword the sentence | The reword is **maintainer-sign-off-gated** (isolated in commit 6) | +| B | Two single-offer definitions exist (`bids_received=1` vs `admitted===1`) | **`bids_received=1`, unified site-wide** — matches the 3 shipped sites; refactor `riskLogic` per-contract flag to match | Us (documented in ADR-0007) | +| C | Composite math for a subject with a missing/thin component | **Mean of *reportable* components** (each with ≥ min-N eligible); band derives from the **count-weighted** composite (robust to one dominant contract); value-weighted shown as context | Us (ADR-0007) | + +Consequence of B to record explicitly: the per-contract `no_competition` flag stops subtracting +rejected bids, so a `3-bid / 2-rejected` contract no longer flags. The "many bids, most rejected" +pattern is **deliberately deferred as its own future flag**, not silently dropped (noted in ADR-0007). + +## 3. Current state (verified against code) + +- **Elementary flags** live only in `apps/web/app/lib/riskLogic.ts` (34 lines, render-time TS), per + contract. Shown via `RiskIndicators.tsx` on `contract.tsx`. No subject-level aggregate exists. +- **Single-offer share is already shipped per authority** — `competition.ts` (`getAuthoritySingleOffer`, + `competitionTotals`) using `bids_received = 1` over a `bids_received >= 1` denominator; also in the + assistant SQL `describe-schema.ts:129,137`. Our composite must agree with these. +- **Rollups** `company_totals` / `authority_totals` are one-row-per-subject, built in + `scripts/precompute.sql` (JOIN-GROUP over `contracts`) and refreshed daily by scoped INSERTs in + `scripts/refresh-slice.sql`. The web reads them via `getCompany`/`getAuthority` (`SELECT *`). +- **D1 bills rows scanned** → aggregates MUST be precomputed, never computed per request. + +## 4. Target architecture (Approach A — materialize once, read many) + +**4.1 Canonical per-contract flags — new columns on `contracts` (nullable = "unknown", never 0):** + +```sql +is_single_offer = CASE WHEN bids_received IS NOT NULL THEN (bids_received = 1) END +is_high_markup = CASE WHEN signing_value_eur IS NOT NULL AND current_value_eur IS NOT NULL + AND signing_value_eur <> 0 + THEN ((current_value_eur - signing_value_eur) / signing_value_eur > 0.2) END +``` + +Populated by ONE unconditional `UPDATE contracts SET …` in `precompute.sql`, inserted **between the +section-0 EUR-timeline UPDATE (line ~40) and the `company_totals` INSERT (line ~50)** — it reads +`signing_value_eur`/`current_value_eur` and must feed the rollups. Mirrored in `refresh-slice.sql` +for the daily path, **after** its recalc UPDATE. + +**4.2 Per-subject aggregates — inline in the GROUP BY INSERT (not correlated subqueries):** +for each component store, per subject: +- `*_k` = flagged count, `*_n` = **eligible** count (own denominator: single-offer → `bids_received>=1`; + high-markup → non-null signing/current). Storing `k`/`n` powers "K от N" (M4) and min-N (M3). +- count-share = `k / NULLIF(n,0)`; value-share = `SUM(flag·amount_eur) / NULLIF(SUM(eligible amount_eur),0)`. +- `composite_count` = mean of reportable components; `band` = CASE over `composite_count`. + +**4.3 Read path:** `getCompany`/`getAuthority` already `SELECT *`, so columns arrive for free — but the +hand-written `*TotalsFull` interfaces and the `CompanyDetail`/`AuthorityDetail` object literals in +`details.ts` must name the new `risk` block. `listCompanies`/`listAuthorities` use explicit `COLS` and +are **not** touched (profile-only; YAGNI). + +## 5. Anti-defamation & data-integrity controls (MANDATORY — from the security review) + +| ID | Control | Implementation | +|---|---|---| +| M2 | No verdict words | Band labels describe the *indicators*: „Малко индикатори" / „Единични индикатори" / „Множество индикатори" / „Много индикатори — заслужава преглед". Never „критичен"/„корупция"/„нередност". | +| M3 | Min-N suppression | A component is reportable only when its **eligible** denominator `n ≥ 5`. If no component is reportable → no band, no score. | +| M4 | Counts beside shares | Always render „34 от 120 договора", never a bare %. | +| M5 | Exclude suspect rows | Reuse the existing `value_flag`/EUR-null rules; a `value_suspect` row can never be `is_high_markup=1` (its EUR figures are NULL). | +| M6 | Concentration guard | Band derives from **count-weighting**; value-weighting is context only, with a note when one contract dominates. | +| M7 | Drill-down | „виж договорите зад този индикатор" → the subject's contracts filtered by the flag. | +| M8 | Atomic block, no leak | Score + disclaimer + counts + link are one `Callout` unit; excluded from ``/OG so it can't become a search snippet. | +| M9 | Constants + persons | Thresholds are server-side constants (never query params); the whole block is **suppressed for natural-person profiles** (reuse `company.tsx:28,51-55`). | + +Reused verbatim: the neutral-indicator disclaimer pattern at `methodology.tsx:359` and Principle #3 +(„СИГМА не тълкува, а показва"). + +**Provisional band cutoffs** (count-weighted composite ∈ [0,1]) — documented as **tunable, pending +calibration against the real distribution**, not presented as science: +`<0.10` Малко · `0.10–0.30` Единични · `0.30–0.55` Множество · `≥0.55` Много — заслужава преглед. + +## 6. Test strategy (TDD-first) + +- **Golden fixture** (`refresh-slice.test.ts` pattern): seed contracts directly, run the SQL, assert + exact shares/composite with `toBeCloseTo(_,6)`. Cover: single-offer true/false incl. NULL bids; + high-markup boundary (`deltaPct = 0.20` → NOT flagged, `0.21` → flagged); suspect rows (NULL EUR → + `is_high_markup` NULL, no error); a subject where **count-share 0.75 ≠ value-share 0.35** (proves the + two weightings diverge); an **authority-side** row (the two rollup blocks are copy-paste twins); + idempotency (re-run → identical); min-N boundary (n=4 suppressed, n=5 shown). +- **Parity guard (no-drift):** narrow `evaluateRiskIndicators`' param to + `RiskFlagInput = Pick` so the test feeds flat SQL-row literals and asserts the + materialized `is_*` column == the TS predicate per row. +- **Full-vs-daily parity:** extend the `refresh-slice.test.ts` projection that compares slice vs full + rebuild to include the new columns (else drift ships green). +- **Non-vacuity:** assert an exact fixture row count, mirroring the `home_totals` guard. + +## 7. Commit plan (ONE PR — dependency order; each commit compiles & its tests pass) + +1. `docs(adr): ADR-0007 subject risk composite` — decisions (grain, 2 components, `bids_received=1` + + the per-contract behavior change, equal weights, count+value, min-N, band cutoffs, neutral + framing, deferred "disqualification-heavy" signal). Pure docs; the design gate. +2. `feat(db): materialize is_single_offer/is_high_markup on contracts` — columns in `0000_init.sql` + **and** the `precompute.sql` mirror; the `UPDATE` in both `precompute.sql` and `refresh-slice.sql`; + flag unit tests. +3. `feat(db): per-subject risk shares + composite on totals` — `k/n`/share/composite/band columns on + both totals tables; inline aggregation in `precompute.sql` + both scoped INSERTs in + `refresh-slice.sql`; golden tests + extended full-vs-slice parity. +4. `refactor(web): unify single-offer on bids_received=1, read materialized flags` — + `ContractDetail` + `details.ts` (row/SELECT/object); `riskLogic` reads the columns; guard the + `deltaPct` crash; rewrite `riskLogic.test.ts`. +5. `feat(web): subject risk indicator on company/authority profiles` — `SubjectRisk` types; + `SubjectRiskIndicator` (own file, score inside a new `neutral` `Callout` variant, visible band + label, ternary-not-`&&`); wire into `company.tsx`/`authority.tsx` via `Section`; M3/M4/M6/M7/M9 guards. +6. `docs(web): methodology section for composite + reword neutrality promise` — new explainer **and** + the `methodology.tsx:146` reword. **Isolated so the maintainer can see exactly what public wording + changes** — merge-gated on their sign-off. + +## 8. Risks + +- **Framing / defamation** (highest) → §5 controls + ADR + maintainer sign-off on the reword. +- **Daily-path drift** (the silent one) → explicit `refresh-slice.sql` steps + extended parity test. +- **Cross-page inconsistency** → resolved by Decision B (`bids_received=1`). +- **`riskLogic` behavior change** → documented; tests updated. +- **Band cutoffs arbitrary** → shipped as provisional/tunable, calibration is a follow-up. + +## 9. Success criteria + +- [ ] Golden + parity + full-vs-slice tests green; `tsc` 0; prettier clean. +- [ ] Composite single-offer share == `getAuthoritySingleOffer` on a shared fixture. +- [ ] No score renders without its disclaimer + counts; suppressed for natural persons and `n<5`. +- [ ] No band label is a verdict word; risk block excluded from ``/OG. +- [ ] Daily refresh keeps the new columns in sync with a full rebuild (parity test proves it). + +## 10. Out of scope (v2 / follow-up) + +Non-procedural/#153 and CPV-cohort/#41+#210 as new elementary flags; decision-window & new-supplier +flags (no data collected); the "disqualification-heavy" signal; band-cutoff calibration against the +real distribution; extracting shared string-scan helpers into `@sigma/shared`. diff --git a/packages/api-contract/src/index.ts b/packages/api-contract/src/index.ts index ad7d6d1d7..d1e690164 100644 --- a/packages/api-contract/src/index.ts +++ b/packages/api-contract/src/index.ts @@ -111,6 +111,18 @@ export interface ConsortiumParticipant { resolvedSlug: string | null; } +/** #229 per-subject risk aggregate — the raw rollup columns (company_totals/authority_totals). The read + * layer (apps/web `subjectRisk.ts`) derives the composite, band, and reportability (min-N) from these, + * so the presentation thresholds live in one place. `*K` = flagged count, `*N` = eligible count. */ +export interface SubjectRiskAggregate { + singleOfferK: number | null; + singleOfferN: number | null; + singleOfferValueShare: number | null; + highMarkupK: number | null; + highMarkupN: number | null; + highMarkupValueShare: number | null; +} + export interface CompanyDetail { slug: string; name: string; @@ -149,6 +161,7 @@ export interface CompanyDetail { * 40 %; 2. … 60 %") rather than a clean `;`-list. Rare (~4 rows in production); rendered as a * quotable block so the original detail survives. */ membershipNote: string | null; + risk: SubjectRiskAggregate | null; } // ── Authorities ───────────────────────────────────────────────────────────────────────────────── @@ -208,6 +221,7 @@ export interface AuthorityDetail { recentContracts: ContractListItem[]; /** Highest-value contracts (listContracts sort='value-desc', amount_eur DESC) — „Най-големи по стойност". */ topContracts: ContractListItem[]; + risk: SubjectRiskAggregate | null; } // ── Contracts ───────────────────────────────────────────────────────────────────────────────── @@ -321,6 +335,11 @@ export interface ContractDetail { euProgramme: string | null; durationDays: number | null; value: ContractValueTimeline; + /** Materialized risk flags (scripts/precompute.sql — the canonical single source the subject-risk + * rollups aggregate). null = not assessable: single-offer needs a known bid count (`bids_received = 1` + * basis), high-markup needs both signing and current EUR figures (suspect rows excluded). */ + isSingleOffer: boolean | null; + isHighMarkup: boolean | null; /** When this contract is one of several awards under the same procedure (more awards than lots — a * framework agreement / dynamic purchasing system call-off), this is the total number of awarded * contracts under the parent tender. Null for a normal single/per-lot award. The procedure-level diff --git a/packages/db/migrations/0000_init.sql b/packages/db/migrations/0000_init.sql index 0888ef93d..f42fbb3ab 100644 --- a/packages/db/migrations/0000_init.sql +++ b/packages/db/migrations/0000_init.sql @@ -1,11 +1,13 @@ -- Sigma — consolidated schema (D1 / SQLite). Single source of truth for the database. -- --- Sigma is pre-production and every import starts from a FRESH database (no deployed data to --- preserve), so the schema is ONE file rather than an incremental migration chain — re-introduce --- incremental migrations only once there is deployed data you cannot drop. Applied by --- `wrangler d1 migrations apply sigma [--local|--remote]`; the full import is `node scripts/import.mjs` --- (work DB: load-eop → derive-amendments → load-fx → load-nuts → normalize-raw → promote-amendments, --- then ship-domain copies the served tables into the served D1 and runs precompute on it). +-- The WORK DB is rebuilt from scratch every import (import.mjs loads THIS file, then load/transform), so +-- 0000_init is its single source of truth. The SERVED D1 (prod) is PERSISTENT: ship-domain runs +-- `wrangler d1 migrations apply`, which is filename-tracked — once applied, 0000_init is frozen there. +-- So add NEW schema objects in a new numbered migration (ALTER TABLE …), NEVER by editing 0000_init: an +-- edit reaches the work DB (and green local CI) but never prod — the applied-migration trap. Touch +-- 0000_init only on a full rebuild that regenerates the whole chain. The full import is +-- `node scripts/import.mjs` (work DB: load-eop → derive-amendments → load-fx → load-nuts → normalize-raw +-- → promote-amendments, then ship-domain copies the served tables into the served D1 + runs precompute). -- -- Modelling rationale (cleaning policy, value_flag, consortium model, canonical EUR + FX, the -- synthetic-tender rule) lives in docs/etl.md and docs/core-scope.md. diff --git a/packages/db/migrations/0006_subject_risk_columns.sql b/packages/db/migrations/0006_subject_risk_columns.sql new file mode 100644 index 000000000..e0e397f5f --- /dev/null +++ b/packages/db/migrations/0006_subject_risk_columns.sql @@ -0,0 +1,31 @@ +-- Subject-risk columns (issue #229) — canonical per-contract flags + per-subject rollups. +-- +-- Added as a NUMBERED migration, NOT by editing 0000_init: the served D1 is persistent and +-- `wrangler d1 migrations apply` is filename-tracked, so an edit to the already-applied 0000_init reaches +-- the work DB (and green local CI) but NEVER prod — the #188/#239 applied-migration trap. New objects go +-- through a new migration; 0000_init is touched only on a full rebuild. +-- +-- SQLite has no `ADD COLUMN IF NOT EXISTS`, so these columns live ONLY here — a fresh DB applies 0000_init +-- (without them) then this file. The `CREATE TABLE IF NOT EXISTS` mirror in scripts/precompute.sql stays a +-- no-op on the existing tables. Number 0006 clears the 0002 claimants (#226/#193/#172) to avoid a +-- duplicate-version at apply; a gap is harmless for filename-tracked application. + +-- contracts: canonical per-contract flags, materialized by scripts/precompute.sql + refresh-slice.sql. +ALTER TABLE contracts ADD COLUMN is_single_offer INTEGER; -- 1/0 = bids_received = 1; NULL = bid count unknown (never counted as 0 by the rollup shares) +ALTER TABLE contracts ADD COLUMN is_high_markup INTEGER; -- 1/0 = (current−signing)/signing > 0.2 on value_flag='ok' rows; NULL = ineligible (suspect / signing≤0 / EUR absent) + +-- company_totals: per-subject risk rollups. Composite + band derived in the read layer (details.ts). +ALTER TABLE company_totals ADD COLUMN single_offer_k INTEGER; -- # flagged single-offer (is_single_offer = 1) +ALTER TABLE company_totals ADD COLUMN single_offer_n INTEGER; -- # eligible (bids_received >= 1) — count-share denominator +ALTER TABLE company_totals ADD COLUMN single_offer_value_share REAL; -- Σ flagged amount_eur / Σ eligible amount_eur; NULL if no eligible value +ALTER TABLE company_totals ADD COLUMN high_markup_k INTEGER; -- # flagged high-markup (is_high_markup = 1) +ALTER TABLE company_totals ADD COLUMN high_markup_n INTEGER; -- # eligible (is_high_markup IS NOT NULL) +ALTER TABLE company_totals ADD COLUMN high_markup_value_share REAL; -- Σ flagged amount_eur / Σ eligible amount_eur; NULL if no eligible value + +-- authority_totals: same rollups per authority (single_offer_n matches getAuthoritySingleOffer). +ALTER TABLE authority_totals ADD COLUMN single_offer_k INTEGER; +ALTER TABLE authority_totals ADD COLUMN single_offer_n INTEGER; +ALTER TABLE authority_totals ADD COLUMN single_offer_value_share REAL; +ALTER TABLE authority_totals ADD COLUMN high_markup_k INTEGER; +ALTER TABLE authority_totals ADD COLUMN high_markup_n INTEGER; +ALTER TABLE authority_totals ADD COLUMN high_markup_value_share REAL; diff --git a/packages/db/src/integrity-checks.test.ts b/packages/db/src/integrity-checks.test.ts index fb2d1aa44..d922e7805 100644 --- a/packages/db/src/integrity-checks.test.ts +++ b/packages/db/src/integrity-checks.test.ts @@ -17,10 +17,12 @@ import { checkNoNegativeValues, checkRollupReconciliation, checkStagingReconciliation, + checkSubjectRiskBounds, } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const riskColumnsPath = resolve(root, 'packages/db/migrations/0006_subject_risk_columns.sql'); const precomputePath = resolve(root, 'scripts/precompute.sql'); function sqlite(dbPath: string, sql: string): void { @@ -61,6 +63,7 @@ function freshDb(): string { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-integrity-')); const dbPath = resolve(dir, 'test.sqlite'); readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); sqlite(dbPath, CLEAN_FIXTURE); return dbPath; } @@ -106,6 +109,7 @@ describe('reconciliation gate — clean corpus', () => { 'eik-validity', 'date-sanity', 'staging-reconciliation', + 'subject-risk-bounds', ]) expect(results.find((r) => r.name === nm)?.skipped, `${nm} must not skip`).toBe(false); }); @@ -116,6 +120,24 @@ describe('reconciliation gate — clean corpus', () => { expect(result.skipped).toBe(true); expect(result.ok).toBe(true); }); + + it('subject-risk-bounds self-skips before precompute (empty rollups)', () => { + const db = track(freshDb()); + const result = checkSubjectRiskBounds(runner(db)); + expect(result.skipped).toBe(true); + expect(result.ok).toBe(true); + }); + + // home_totals predates the risk columns, so it is not proof they exist; a DB with a populated + // home_totals but no risk column must skip, not throw 'no such column'. + it('subject-risk-bounds self-skips (not throws) when a risk column is missing but home_totals exists', () => { + const db = track(freshDb()); + precompute(db); // populates home_totals AND the risk columns + sqlite(db, 'ALTER TABLE contracts DROP COLUMN is_high_markup;'); // drift: column gone + const result = checkSubjectRiskBounds(runner(db)); + expect(result.skipped).toBe(true); + expect(result.ok).toBe(true); + }); }); describe('reconciliation gate — injected violations', () => { @@ -259,6 +281,39 @@ describe('reconciliation gate — injected violations', () => { expect(result.ok).toBe(true); }); + it('subject-risk-bounds catches a value_share above 1 (the Finding-1 200% bug)', () => { + const db = track(freshDb()); + precompute(db); + sqlite( + db, + "UPDATE company_totals SET single_offer_value_share = 2.0 WHERE bidder_id = 'eik:131071587';", + ); + const result = checkSubjectRiskBounds(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/single_offer_value_share outside \[0,1\]/); + }); + + it('subject-risk-bounds catches a flagged count exceeding its denominator (k > n)', () => { + const db = track(freshDb()); + precompute(db); + sqlite( + db, + "UPDATE company_totals SET single_offer_k = single_offer_n + 1 WHERE bidder_id = 'eik:131071587';", + ); + const result = checkSubjectRiskBounds(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/single_offer_k > single_offer_n/); + }); + + it('subject-risk-bounds catches is_high_markup set on a non-ok (suspect) contract', () => { + const db = track(freshDb()); + precompute(db); + sqlite(db, "UPDATE contracts SET is_high_markup = 1, value_flag = 'review' WHERE id = 'c:1';"); + const result = checkSubjectRiskBounds(runner(db)); + expect(result.ok).toBe(false); + expect(result.detail).toMatch(/is_high_markup set on a non-'ok' value_flag/); + }); + it('assertIntegrity throws non-zero on a sign-flipped amount_eur (the import would exit 1)', () => { const db = track(freshDb()); precompute(db); diff --git a/packages/db/src/queries/contracts-filter-sql.test.ts b/packages/db/src/queries/contracts-filter-sql.test.ts index 2c4d69746..f2d50819a 100644 --- a/packages/db/src/queries/contracts-filter-sql.test.ts +++ b/packages/db/src/queries/contracts-filter-sql.test.ts @@ -29,11 +29,11 @@ INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) VALUES ('eik:200000001', 'Фирма Х', '200000001', '200000001', 1, 'company'); INSERT INTO tenders (id, source_id, title, authority_id, cpv_code, procedure_type, status) VALUES ('t:A', 'UNP-A', 'Поръчка А', 'auth:100000001', '45000000', 'открита процедура', 'awarded'); -INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, bids_received, value_flag, amount_eur) VALUES - ('c:NULL', 't:A', 'eik:200000001', 100, 'EUR', '2024-01-01', NULL, 'ok', 100), - ('c:ZERO', 't:A', 'eik:200000001', 200, 'EUR', '2024-01-02', 0, 'ok', 200), - ('c:ONE', 't:A', 'eik:200000001', 300, 'EUR', '2024-01-03', 1, 'ok', 300), - ('c:TWO', 't:A', 'eik:200000001', 400, 'EUR', '2024-01-04', 2, 'ok', 400); +INSERT INTO contracts (id, tender_id, bidder_id, amount, currency, signed_at, bids_received, value_flag, amount_eur, is_high_markup) VALUES + ('c:NULL', 't:A', 'eik:200000001', 100, 'EUR', '2024-01-01', NULL, 'ok', 100, NULL), + ('c:ZERO', 't:A', 'eik:200000001', 200, 'EUR', '2024-01-02', 0, 'ok', 200, 0), + ('c:ONE', 't:A', 'eik:200000001', 300, 'EUR', '2024-01-03', 1, 'ok', 300, 0), + ('c:TWO', 't:A', 'eik:200000001', 400, 'EUR', '2024-01-04', 2, 'ok', 400, 1); `; /** Minimal D1Database facade over node:sqlite — enough for the query layer's prepare/bind/all/first. */ @@ -113,4 +113,14 @@ describe('contract filters against a real SQLite engine (#138)', () => { expect(page.total).toBe(1); expect(page.items[0]!.bidsReceived).toBe(1); }); + + it('markup=high narrows the list to exactly the flagged row', async () => { + const db = realDb(); + + // Seed flags is_high_markup on c:TWO only; NULL (unassessable) and 0 rows must be excluded. + const flagged = await listContracts(db, { markup: 'high', pageSize: 10 }); + expect(flagged.total).toBe(1); + expect(flagged.items).toHaveLength(1); + expect(flagged.items[0]!.valueEur).toBe(400); + }); }); diff --git a/packages/db/src/queries/contracts.ts b/packages/db/src/queries/contracts.ts index 4138c9273..52bfdb0c7 100644 --- a/packages/db/src/queries/contracts.ts +++ b/packages/db/src/queries/contracts.ts @@ -30,6 +30,7 @@ export interface ContractListParams { bidder?: string | null; // bidder slug q?: string | null; bids?: 'one' | null; + markup?: 'high' | null; cursor?: string | null; pageSize?: number; } @@ -44,6 +45,7 @@ export const CONTRACT_FILTER_KEYS = [ 'bidder', 'q', 'bids', + 'markup', ] as const satisfies readonly (keyof ContractListParams)[]; // Compile-time completeness guard (issue #138 bug class) — see filter-guard.ts. If this line @@ -157,6 +159,7 @@ function buildFilters(p: ContractListParams): { sql: string; params: unknown[] } if (p.eu === 'eu') where.push(`c.eu_funded = 1`); else if (p.eu === 'national') where.push(`(c.eu_funded IS NULL OR c.eu_funded = 0)`); if (p.bids === 'one') where.push(`c.bids_received = 1`); + if (p.markup === 'high') where.push(`c.is_high_markup = 1`); if (p.authority) { where.push(`t.authority_id = ?`); params.push('auth:' + p.authority); @@ -192,6 +195,7 @@ function contractFilterSignature(p: ContractListParams): string { bidder, q: searchMatchQuery(p.q ?? ''), bids: p.bids ?? null, + markup: p.markup ?? null, } satisfies Record<(typeof CONTRACT_FILTER_KEYS)[number], unknown>; return filterSignature(filters); } diff --git a/packages/db/src/queries/details.ts b/packages/db/src/queries/details.ts index 8b3604f5e..6d580c24b 100644 --- a/packages/db/src/queries/details.ts +++ b/packages/db/src/queries/details.ts @@ -7,6 +7,7 @@ import type { AuthorityShare, BidDistribution, CompanyDetail, + SubjectRiskAggregate, CompanyShare, ConsortiumParticipant, ContractDetail, @@ -19,7 +20,12 @@ import type { SectorSpend, } from '@sigma/api-contract'; import { CPV_SECTORS, PROCEDURE_GROUPS, procedureGroup } from '@sigma/config'; -import { cleanName, entityName, parseConsortiumMembers } from '@sigma/shared'; +import { + cleanName, + entityName, + isNaturalPersonSubject, + parseConsortiumMembers, +} from '@sigma/shared'; import { listContracts } from './contracts'; import { authoritySlug, companySlug, contractSlug } from './identity'; import { typeLabel } from './rows'; @@ -91,6 +97,40 @@ interface CompanyTotalsFull { eu_eur: number; first_date: string | null; last_date: string | null; + single_offer_k: number | null; + single_offer_n: number | null; + single_offer_value_share: number | null; + high_markup_k: number | null; + high_markup_n: number | null; + high_markup_value_share: number | null; +} + +// Map the raw subject-risk rollup columns (shared by company_totals/authority_totals) to the DTO aggregate; the +// read layer's subjectRisk.ts derives the composite/band/reportability from these. +function subjectRiskAggregate(row: { + single_offer_k: number | null; + single_offer_n: number | null; + single_offer_value_share: number | null; + high_markup_k: number | null; + high_markup_n: number | null; + high_markup_value_share: number | null; +}): SubjectRiskAggregate { + const { + single_offer_k, + single_offer_n, + single_offer_value_share, + high_markup_k, + high_markup_n, + high_markup_value_share, + } = row; + return { + singleOfferK: single_offer_k, + singleOfferN: single_offer_n, + singleOfferValueShare: single_offer_value_share, + highMarkupK: high_markup_k, + highMarkupN: high_markup_n, + highMarkupValueShare: high_markup_value_share, + }; } export async function getCompany(db: D1Database, bidderId: string): Promise { @@ -179,6 +219,14 @@ export async function getCompany(db: D1Database, bidderId: string): Promise = { countBucket: '2-5', eu: 'eu', kinds: ['company'], + markup: 'high', procedureGroups: ['open'], q: 'rail', sectors: ['45'], @@ -76,6 +77,7 @@ describe('route filter signatures', () => { 'bidder', 'q', 'bids', + 'markup', ]); expect([...COMPANY_FILTER_KEYS]).toEqual([ 'kinds', diff --git a/packages/db/src/refresh-slice.test.ts b/packages/db/src/refresh-slice.test.ts index aa20ec76f..d5ded7d6f 100644 --- a/packages/db/src/refresh-slice.test.ts +++ b/packages/db/src/refresh-slice.test.ts @@ -9,8 +9,10 @@ import { assertIntegrity } from '../../../scripts/integrity-checks.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const riskColumnsPath = resolve(root, 'packages/db/migrations/0006_subject_risk_columns.sql'); const refreshSlicePath = resolve(root, 'scripts/refresh-slice.sql'); const normalizePath = resolve(root, 'scripts/normalize-raw.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); const workStagingSchemaPath = resolve(root, 'scripts/work-staging-schema.sql'); function sqlite(dbPath: string, sql: string): string { @@ -175,6 +177,7 @@ function seedOcdsOnlySharedNumber(dbPath: string): void { function initWorkDb(dbPath: string): void { readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); readScript(dbPath, workStagingSchemaPath); } @@ -358,6 +361,7 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); readScript(dbPath, workStagingSchemaPath); seedEopBaseDay(dbPath); @@ -437,6 +441,7 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); readScript(dbPath, workStagingSchemaPath); seedEopOnlySharedNumber(dbPath); readScript(dbPath, refreshSlicePath); @@ -485,6 +490,7 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -533,6 +539,7 @@ describe('refresh-slice EOP base derivation', () => { const dbPath = resolve(dir, 'test.sqlite'); try { readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); readScript(dbPath, workStagingSchemaPath); sqlite( dbPath, @@ -642,6 +649,46 @@ describe('refresh-slice EOP base derivation', () => { } }); + // #229 drift guard: the daily slice path (refresh-slice.sql) must materialize the same per-contract + // flags and per-subject risk aggregates as a full rebuild (normalize-raw + precompute). If a column is + // added to one path and not the other, this fails instead of silently shipping stale risk numbers. + it('materializes identical risk flags and aggregates on the slice and full paths (#229)', () => { + const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); + const fullDb = resolve(dir, 'full.sqlite'); + const sliceDb = resolve(dir, 'slice.sqlite'); + try { + initWorkDb(fullDb); + initWorkDb(sliceDb); + seedContractIdFixture(fullDb); + seedContractIdFixture(sliceDb); + + readScript(fullDb, normalizePath); + readScript(fullDb, precomputePath); + readScript(sliceDb, refreshSlicePath); + + const flagsSql = 'SELECT id, is_single_offer, is_high_markup FROM contracts ORDER BY id'; + const fullFlags = sqliteJson<{ + id: string; + is_single_offer: number | null; + is_high_markup: number | null; + }>(fullDb, flagsSql); + expect(sqliteJson(sliceDb, flagsSql)).toEqual(fullFlags); + // non-vacuity: the fixture actually exercises the single-offer flag (not an all-NULL vacuous pass). + expect(fullFlags.some((row) => row.is_single_offer !== null)).toBe(true); + + const riskCols = + 'single_offer_k, single_offer_n, ROUND(single_offer_value_share, 6) AS so_vs, ' + + 'high_markup_k, high_markup_n, ROUND(high_markup_value_share, 6) AS hm_vs'; + const companySql = `SELECT bidder_id, ${riskCols} FROM company_totals ORDER BY bidder_id`; + expect(sqliteJson(sliceDb, companySql)).toEqual(sqliteJson(fullDb, companySql)); + + const authoritySql = `SELECT authority_id, ${riskCols} FROM authority_totals ORDER BY authority_id`; + expect(sqliteJson(sliceDb, authoritySql)).toEqual(sqliteJson(fullDb, authoritySql)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('keeps contract ids stable across post-amendment and pre-amendment staging', () => { const dir = mkdtempSync(resolve(tmpdir(), 'sigma-refresh-slice-')); const fullDb = resolve(dir, 'full.sqlite'); diff --git a/packages/db/src/risk-flags.test.ts b/packages/db/src/risk-flags.test.ts new file mode 100644 index 000000000..c80736088 --- /dev/null +++ b/packages/db/src/risk-flags.test.ts @@ -0,0 +1,116 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const riskColumnsPath = resolve(root, 'packages/db/migrations/0006_subject_risk_columns.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +// One tender/bidder/authority parent, then contracts seeded straight into the domain table (bypassing +// raw_*/normalize) with the exact bid counts and signing/current values each flag case needs. EUR +// currency so the section-0 EUR timeline copies the figures as-is and the 0.20-vs-0.21 markup boundary +// is exact (no BGN-peg float noise). One precompute run covers every case. +let dir: string; +let dbPath: string; + +beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-risk-flags-')); + dbPath = resolve(dir, 'test.sqlite'); + readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; +INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:1', 'A', '100000001', 'public'); +INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) + VALUES ('eik:1', 'B', '200000001', '200000001', 1, 'company'); +INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) + VALUES ('t:1', 'UNP-1', 'T', 'auth:1', 1000, 'EUR', 'open', 'awarded'); +INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, signing_value, current_value, value_flag, bids_received) +VALUES + ('c:so1', 't:1', 'eik:1', 1000, 'EUR', NULL, NULL, 'ok', 1), + ('c:so3', 't:1', 'eik:1', 1000, 'EUR', NULL, NULL, 'ok', 3), + ('c:soN', 't:1', 'eik:1', 1000, 'EUR', NULL, NULL, 'ok', NULL), + ('c:hmB', 't:1', 'eik:1', 1000, 'EUR', 1000, 1200, 'ok', 2), + ('c:hm1', 't:1', 'eik:1', 1000, 'EUR', 1000, 1210, 'ok', 2), + ('c:hmS', 't:1', 'eik:1', 1000, 'EUR', 1000, 5000, 'value_suspect', 2), + ('c:hm0', 't:1', 'eik:1', 1000, 'EUR', 1000, 1000, 'ok', 2), + ('c:hmR', 't:1', 'eik:1', 1000, 'EUR', 1000, 1400, 'review', 2), + ('c:hmL', 't:1', 'eik:1', 1000, 'EUR', 1000, 1400, 'value_low', 2);`, + ); + readScript(dbPath, precomputePath); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function flag(id: string, col: 'is_single_offer' | 'is_high_markup'): number | null { + const row = sqliteJson>( + dbPath, + `SELECT ${col} FROM contracts WHERE id = '${id}'`, + )[0]; + return row?.[col] ?? null; +} + +describe('per-contract risk flags (#229, precompute)', () => { + it('flags single-offer when bids_received = 1', () => { + expect(flag('c:so1', 'is_single_offer')).toBe(1); + }); + + it('does not flag single-offer when more than one bid was received', () => { + expect(flag('c:so3', 'is_single_offer')).toBe(0); + }); + + it('leaves single-offer NULL when the bid count is unknown', () => { + expect(flag('c:soN', 'is_single_offer')).toBe(null); + }); + + it('does not flag high-markup at the 20% boundary (strictly greater than)', () => { + expect(flag('c:hmB', 'is_high_markup')).toBe(0); + }); + + it('flags high-markup above 20%', () => { + expect(flag('c:hm1', 'is_high_markup')).toBe(1); + }); + + it('leaves high-markup NULL for a value-suspect row (no trustworthy EUR figures)', () => { + expect(flag('c:hmS', 'is_high_markup')).toBe(null); + }); + + it('does not flag high-markup with no markup', () => { + expect(flag('c:hm0', 'is_high_markup')).toBe(0); + }); + + // The contract page marks review/value_low rows as suspect and hides the badge; the flag must match + // (else the rollup counts a markup the page won't show). #229 review finding. + it('leaves high-markup NULL for a review row despite a >20% markup', () => { + expect(flag('c:hmR', 'is_high_markup')).toBe(null); + }); + + it('leaves high-markup NULL for a value_low row despite a >20% markup', () => { + expect(flag('c:hmL', 'is_high_markup')).toBe(null); + }); +}); diff --git a/packages/db/src/risk-rollups.test.ts b/packages/db/src/risk-rollups.test.ts new file mode 100644 index 000000000..304eb6416 --- /dev/null +++ b/packages/db/src/risk-rollups.test.ts @@ -0,0 +1,153 @@ +/// +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const schemaPath = resolve(root, 'packages/db/migrations/0000_init.sql'); +const riskColumnsPath = resolve(root, 'packages/db/migrations/0006_subject_risk_columns.sql'); +const precomputePath = resolve(root, 'scripts/precompute.sql'); + +function sqlite(dbPath: string, sql: string): void { + execFileSync('sqlite3', [dbPath], { input: sql, encoding: 'utf8' }); +} + +function sqliteJson(dbPath: string, sql: string): T[] { + const out = execFileSync('sqlite3', ['-json', dbPath, sql], { encoding: 'utf8' }).trim(); + return out ? (JSON.parse(out) as T[]) : []; +} + +function readScript(dbPath: string, path: string): void { + execFileSync('sqlite3', [dbPath], { + input: `PRAGMA foreign_keys=ON;\n.read ${path}\n`, + stdio: 'pipe', + }); +} + +// One bidder (eik:1) and one authority (auth:1) over the SAME five contracts, so the company- and +// authority-side aggregates must come out identical. Chosen so the count-share (0.75) and value-share +// (0.35) genuinely diverge, and so the NULL-flag contract (K5) is excluded from the risk denominators +// even though it counts toward `contracts` — proving the denominators are not `contracts`. +// +// id bids signing current value_flag amount_eur is_single_offer is_high_markup +// K1 1 1000 1000 ok 1000 1 0 +// K2 1 1000 1300 ok 2000 1 1 (deltaPct 0.30) +// K3 1 1000 1000 ok 500 1 0 +// K4 3 1000 1000 ok 6500 0 0 +// K5 NULL 1000 5000 value_suspect 1000 NULL (bids) NULL (suspect EUR) +// +// single-offer: k=3, n=4 (K1..K4) → count 0.75 ; value 3500/10000 = 0.35 +// high-markup : k=1, n=4 (K1..K4) → count 0.25 ; value 2000/10000 = 0.20 +interface RiskRow { + single_offer_k: number; + single_offer_n: number; + single_offer_value_share: number; + high_markup_k: number; + high_markup_n: number; + high_markup_value_share: number; + contracts: number; +} + +let dir: string; +let dbPath: string; + +function riskRow(table: 'company_totals' | 'authority_totals', key: string): RiskRow { + const col = table === 'company_totals' ? 'bidder_id' : 'authority_id'; + return sqliteJson( + dbPath, + `SELECT single_offer_k, single_offer_n, single_offer_value_share, + high_markup_k, high_markup_n, high_markup_value_share, contracts + FROM ${table} WHERE ${col} = '${key}'`, + )[0]!; +} + +beforeAll(() => { + dir = mkdtempSync(resolve(tmpdir(), 'sigma-risk-rollups-')); + dbPath = resolve(dir, 'test.sqlite'); + readScript(dbPath, schemaPath); + readScript(dbPath, riskColumnsPath); + sqlite( + dbPath, + `PRAGMA foreign_keys=ON; +INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:1', 'A', '100000001', 'public'); +INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) + VALUES ('eik:1', 'B', '200000001', '200000001', 1, 'company'); +INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) + VALUES ('t:1', 'UNP-1', 'T', 'auth:1', 1000, 'EUR', 'open', 'awarded'); +INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, signing_value, current_value, value_flag, bids_received, amount_eur) +VALUES + ('c:K1', 't:1', 'eik:1', 1000, 'EUR', 1000, 1000, 'ok', 1, 1000), + ('c:K2', 't:1', 'eik:1', 2000, 'EUR', 1000, 1300, 'ok', 1, 2000), + ('c:K3', 't:1', 'eik:1', 500, 'EUR', 1000, 1000, 'ok', 1, 500), + ('c:K4', 't:1', 'eik:1', 6500, 'EUR', 1000, 1000, 'ok', 3, 6500), + ('c:K5', 't:1', 'eik:1', 1000, 'EUR', 1000, 5000, 'value_suspect', NULL, 1000); +-- Adversarial (#229 review): a value_low contract carries a NEGATIVE amount_eur (normalize keeps +-- zero/negative rows in the sums). The value share must stay within [0,1] — proven by weighting only +-- positive amount_eur. eik:2 single-offer value share = 300 / (300 + 100) = 0.75, NOT (300-500)/(300-500+100). +INSERT INTO authorities (id, name, bulstat, type) VALUES ('auth:2', 'A2', '100000002', 'public'); +INSERT INTO bidders (id, name, bulstat, eik_normalized, eik_valid, kind) + VALUES ('eik:2', 'B2', '200000002', '200000002', 1, 'company'); +INSERT INTO tenders (id, source_id, title, authority_id, estimated_value, currency, procedure_type, status) + VALUES ('t:2', 'UNP-2', 'T2', 'auth:2', 1000, 'EUR', 'open', 'awarded'); +INSERT INTO contracts + (id, tender_id, bidder_id, amount, currency, signing_value, current_value, value_flag, bids_received, amount_eur) +VALUES + ('c:V1', 't:2', 'eik:2', 300, 'EUR', 300, 300, 'ok', 1, 300), + ('c:V2', 't:2', 'eik:2', -500, 'EUR', -500, -500, 'value_low', 1, -500), + ('c:V3', 't:2', 'eik:2', 100, 'EUR', 100, 100, 'ok', 3, 100);`, + ); + readScript(dbPath, precomputePath); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('per-subject risk aggregates (#229, precompute)', () => { + it('counts single-offer contracts over the known-bid denominator (K of N)', () => { + const r = riskRow('company_totals', 'eik:1'); + expect(r.single_offer_k).toBe(3); + expect(r.single_offer_n).toBe(4); + }); + + it('weights the single-offer share by value, diverging from the count share', () => { + // count share = 3/4 = 0.75; value share = 3500/10000 = 0.35 — deliberately different. + expect(riskRow('company_totals', 'eik:1').single_offer_value_share).toBeCloseTo(0.35, 6); + }); + + it('excludes the value-suspect row from the high-markup denominator', () => { + const r = riskRow('company_totals', 'eik:1'); + expect(r.high_markup_k).toBe(1); + expect(r.high_markup_n).toBe(4); + }); + + it('weights the high-markup share by value', () => { + expect(riskRow('company_totals', 'eik:1').high_markup_value_share).toBeCloseTo(0.2, 6); + }); + + it('excludes NULL-flag contracts from the risk denominators (n below contracts)', () => { + const r = riskRow('company_totals', 'eik:1'); + expect(r.contracts).toBe(5); + expect(r.single_offer_n).toBe(4); + expect(r.high_markup_n).toBe(4); + }); + + it('produces identical aggregates on the authority side', () => { + expect(riskRow('authority_totals', 'auth:1')).toEqual(riskRow('company_totals', 'eik:1')); + }); + + it('keeps the value share within [0,1] despite a negative value_low contract', () => { + // Without the positive-money guard this is (300-500)/(300-500+100) = 2.0 — a 200% "share". + expect(riskRow('company_totals', 'eik:2').single_offer_value_share).toBeCloseTo(0.75, 6); + }); + + it('is idempotent — a second precompute run yields the same aggregates', () => { + const before = riskRow('company_totals', 'eik:1'); + readScript(dbPath, precomputePath); + expect(riskRow('company_totals', 'eik:1')).toEqual(before); + }); +}); diff --git a/packages/shared/src/format.test.ts b/packages/shared/src/format.test.ts index cbbd63669..e6f35070e 100644 --- a/packages/shared/src/format.test.ts +++ b/packages/shared/src/format.test.ts @@ -5,6 +5,7 @@ import { date, entityName, isNaturalPersonProfileName, + isNaturalPersonSubject, longDate, money, moneyBare, @@ -185,3 +186,29 @@ describe('isNaturalPersonProfileName', () => { expect(isNaturalPersonProfileName('СОФАРМА ТРЕЙДИНГ АД')).toBe(false); }); }); + +describe('isNaturalPersonSubject', () => { + it('flags a sole-trader legal form', () => { + expect( + isNaturalPersonSubject({ kind: 'company', legalForm: 'ЕТ', displayName: 'ФИРМА ООД' }), + ).toBe(true); + }); + + it('flags a sole-trader name even when the legal form is absent', () => { + expect( + isNaturalPersonSubject({ kind: 'company', legalForm: null, displayName: 'ЕТ ДРИФТ - ИВАН' }), + ).toBe(true); + }); + + it('does not flag an ordinary company', () => { + expect( + isNaturalPersonSubject({ kind: 'company', legalForm: 'ООД', displayName: 'СОФАРМА АД' }), + ).toBe(false); + }); + + it('never flags a consortium, whatever its legal form', () => { + expect( + isNaturalPersonSubject({ kind: 'consortium', legalForm: 'ЕТ', displayName: 'ОБЕДИНЕНИЕ' }), + ).toBe(false); + }); +}); diff --git a/packages/shared/src/format.ts b/packages/shared/src/format.ts index 895dd971f..60a5e2264 100644 --- a/packages/shared/src/format.ts +++ b/packages/shared/src/format.ts @@ -198,6 +198,36 @@ export function isNaturalPersonProfileName(name: string): boolean { return normalized.startsWith('ЕТ ') || normalized.startsWith('ET '); } +/** True when a sole-trader legal form marks the profile as one natural person (ЕТ / едноличен търговец). */ +function isSoleTraderLegalForm(kind: string, legalForm: string | null): boolean { + if (kind === 'consortium' || !legalForm) return false; + const normalized = legalForm.trim().toUpperCase(); + return ( + normalized === 'ЕТ' || + normalized === 'ET' || + normalized.includes('ЕДНОЛИЧЕН ТЪРГОВЕЦ') || + normalized.includes('SOLE TRADER') || + normalized.includes('INDIVIDUAL') + ); +} + +/** + * True when a bidder profile denotes a single natural person — by sole-trader legal form or by name. + * Risk indicators and search indexing are suppressed for these so the platform never profiles a named + * individual (the natural-person control, M9). The single home for that determination, shared by the + * company route and the DB read layer, so the suppression can never drift between them. + */ +export function isNaturalPersonSubject(subject: { + kind: string; + legalForm: string | null; + displayName: string; +}): boolean { + return ( + isSoleTraderLegalForm(subject.kind, subject.legalForm) || + isNaturalPersonProfileName(subject.displayName) + ); +} + /** * Display name for a winning entity. A consortium row holds a `;`-joined member list → show the * first member + „и др." (the **Обединение** badge is rendered separately by the caller). Companies diff --git a/scripts/integrity-checks.d.mts b/scripts/integrity-checks.d.mts index 6b2bd0f28..b60cbdd57 100644 --- a/scripts/integrity-checks.d.mts +++ b/scripts/integrity-checks.d.mts @@ -26,6 +26,7 @@ export function checkNoNegativeValues(runner: IntegrityRunner): IntegrityResult; export function checkEikValidity(runner: IntegrityRunner): IntegrityResult; export function checkDateSanity(runner: IntegrityRunner): IntegrityResult; export function checkStagingReconciliation(runner: IntegrityRunner): IntegrityResult; +export function checkSubjectRiskBounds(runner: IntegrityRunner): IntegrityResult; export const CHECKS: Array<(runner: IntegrityRunner) => IntegrityResult>; export function runIntegrityChecks(runner: IntegrityRunner): IntegrityResult[]; diff --git a/scripts/integrity-checks.mjs b/scripts/integrity-checks.mjs index 56ab0e5ab..6c953d7cd 100644 --- a/scripts/integrity-checks.mjs +++ b/scripts/integrity-checks.mjs @@ -46,6 +46,13 @@ function tableExists(runner, name) { ); } +function columnExists(runner, table, column) { + return ( + rows(runner, `SELECT name FROM pragma_table_info('${table}') WHERE name = '${column}'`).length > + 0 + ); +} + // 0) Non-empty corpus — UNCONDITIONAL hard guard. A catastrophic upstream failure (0 candidates) or // a botched derive can leave 0 contracts. On the served D1 the staging check self-skips (no // pipeline_stats) and every rollup sum is 0 == 0, so without this an empty database would pass the @@ -320,6 +327,70 @@ export function checkStagingReconciliation(runner) { }; } +// 6) Subject-risk aggregate bounds. The per-subject shares are ratios that MUST stay in [0,1], +// and each flagged count must not exceed its eligible denominator (single_offer_k ⊆ single_offer_n by +// construction — bids=1 ⇒ bids≥1; likewise high-markup). A value_share outside [0,1] is a computation +// bug — the exact class fixed pre-merge, where a negative value_low amount_eur leaked into the value +// weighting and produced a 200% share. NULL is allowed (unassessable component). Self-skips until +// precompute has written the rollups (home_totals row present). +export function checkSubjectRiskBounds(runner) { + const name = 'subject-risk-bounds'; + // Gate on the risk columns EXISTING (structural), not just home_totals presence: home_totals predates + // these columns, so a drifted DB with a populated home_totals but no risk columns would otherwise throw + // 'no such column' instead of skipping. Skip cleanly on any DB that hasn't got them yet. + if ( + !tableExists(runner, 'home_totals') || + num(scalar(runner, 'SELECT COUNT(*) AS n FROM home_totals', 'n')) === 0 || + !columnExists(runner, 'company_totals', 'single_offer_k') || + !columnExists(runner, 'contracts', 'is_high_markup') + ) { + return { + name, + ok: true, + skipped: true, + detail: 'subject-risk columns absent (schema without them, or rollups not built)', + }; + } + const fails = []; + for (const table of ['company_totals', 'authority_totals']) { + const r = + rows( + runner, + 'SELECT' + + ` (SELECT COUNT(*) FROM ${table} WHERE single_offer_value_share IS NOT NULL AND (single_offer_value_share < 0 OR single_offer_value_share > 1)) AS so_vs,` + + ` (SELECT COUNT(*) FROM ${table} WHERE high_markup_value_share IS NOT NULL AND (high_markup_value_share < 0 OR high_markup_value_share > 1)) AS hm_vs,` + + ` (SELECT COUNT(*) FROM ${table} WHERE single_offer_k > single_offer_n) AS so_kn,` + + ` (SELECT COUNT(*) FROM ${table} WHERE high_markup_k > high_markup_n) AS hm_kn`, + )[0] || {}; + if (num(r.so_vs) !== 0) + fails.push(`${num(r.so_vs)} ${table} single_offer_value_share outside [0,1]`); + if (num(r.hm_vs) !== 0) + fails.push(`${num(r.hm_vs)} ${table} high_markup_value_share outside [0,1]`); + if (num(r.so_kn) !== 0) fails.push(`${num(r.so_kn)} ${table} single_offer_k > single_offer_n`); + if (num(r.hm_kn) !== 0) fails.push(`${num(r.hm_kn)} ${table} high_markup_k > high_markup_n`); + } + // is_high_markup must match the contract page's suspect rule: only value_flag='ok' rows are eligible + // (review/value_low/*_suspect hide the badge on the contract page). A flag set on a non-'ok' row would + // inflate the composite band above what any contract actually displays. + const suspectMarkup = num( + scalar( + runner, + "SELECT COUNT(*) AS n FROM contracts WHERE is_high_markup IS NOT NULL AND value_flag <> 'ok'", + 'n', + ), + ); + if (suspectMarkup !== 0) + fails.push(`${suspectMarkup} contracts have is_high_markup set on a non-'ok' value_flag`); + return { + name, + ok: fails.length === 0, + skipped: false, + detail: fails.length + ? fails.join('; ') + : "subject-risk shares in [0,1], k <= n, high-markup only on 'ok' rows", + }; +} + export const CHECKS = [ checkNonEmptyCorpus, checkRollupReconciliation, @@ -327,6 +398,7 @@ export const CHECKS = [ checkEikValidity, checkDateSanity, checkStagingReconciliation, + checkSubjectRiskBounds, ]; export function runIntegrityChecks(runner) { diff --git a/scripts/precompute.sql b/scripts/precompute.sql index d52642d16..9e74707de 100644 --- a/scripts/precompute.sql +++ b/scripts/precompute.sql @@ -39,6 +39,24 @@ UPDATE contracts SET WHEN fx_rate IS NOT NULL THEN current_value * fx_rate ELSE NULL END; +-- ── 0b) Per-contract risk flags (#229) ───────────────────────────────────────────────────────── +-- Canonical single source for the two elementary flags the subject-risk rollups aggregate, and the +-- flags riskLogic.ts reads on the contract page. NULL = "unknown/ineligible" (the rollup shares drop +-- NULL from BOTH numerator and denominator — never count it as 0). single-offer basis is +-- bids_received = 1, the same basis as competition.ts / describe-schema (ADR-0007). high-markup requires +-- value_flag = 'ok' AND a positive signing EUR, matching the contract page's suspect rule exactly: +-- value_flag='ok' is the EXACT complement of the suspect set {review, value_low, value_suspect, +-- annex_suspect} that details.ts hides — so the rollup never counts a markup the page won't show, and a +-- negative signing baseline can't invert the ratio's sign. Adding a new value_flag variant? Re-check this +-- gate — 'ok' must stay the not-suspect complement or the rollup and the page silently drift apart. +-- Unconditional UPDATE (no WHERE) so a re-run recomputes every row and clears stale values. +UPDATE contracts SET + is_single_offer = CASE WHEN bids_received IS NOT NULL THEN (bids_received = 1) END, + is_high_markup = CASE WHEN value_flag = 'ok' + AND signing_value_eur IS NOT NULL AND current_value_eur IS NOT NULL + AND signing_value_eur > 0 + THEN ((current_value_eur - signing_value_eur) / signing_value_eur > 0.2) END; + -- ── 1) home_totals shell (filled after company/authority rollups exist) ────────────────────────── CREATE TABLE IF NOT EXISTS home_totals ( id INTEGER PRIMARY KEY CHECK (id = 1), contracts INTEGER NOT NULL, value_eur REAL NOT NULL, @@ -52,7 +70,9 @@ CREATE TABLE IF NOT EXISTS company_totals ( bidder_id TEXT PRIMARY KEY REFERENCES bidders(id), name TEXT NOT NULL, kind TEXT NOT NULL, ownership_kind TEXT, eik TEXT, eik_valid INTEGER NOT NULL DEFAULT 0, settlement TEXT, won_eur REAL NOT NULL, contracts INTEGER NOT NULL, authorities INTEGER NOT NULL, primary_sector TEXT, - eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT + eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT, + single_offer_k INTEGER, single_offer_n INTEGER, single_offer_value_share REAL, + high_markup_k INTEGER, high_markup_n INTEGER, high_markup_value_share REAL ); DELETE FROM company_totals; INSERT INTO company_totals (bidder_id, name, kind, ownership_kind, eik, eik_valid, settlement, won_eur, contracts, authorities, eu_eur, first_date, last_date) @@ -68,13 +88,36 @@ UPDATE company_totals SET primary_sector = ( SELECT substr(t.cpv_code, 1, 2) FROM contracts c JOIN tenders t ON t.id = c.tender_id WHERE c.bidder_id = company_totals.bidder_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1); +-- Per-subject risk aggregates. Each component's denominator is its own eligible universe (by design): +-- single_offer_n = ≥1-bid contracts (excludes 0-bid/failed; matches competition.ts); high_markup_n = +-- value-assessable contracts (is_high_markup IS NOT NULL). Value shares weight POSITIVE amount_eur only, +-- so a value_low ≤0 row can't push a share out of [0,1]. A NULL/zero eligible denominator → NULL share. +UPDATE company_totals SET + single_offer_k = agg.so_k, single_offer_n = agg.so_n, single_offer_value_share = agg.so_vshare, + high_markup_k = agg.hm_k, high_markup_n = agg.hm_n, high_markup_value_share = agg.hm_vshare +FROM ( + SELECT c.bidder_id, + SUM(CASE WHEN c.is_single_offer = 1 THEN 1 ELSE 0 END) AS so_k, + SUM(CASE WHEN c.bids_received >= 1 THEN 1 ELSE 0 END) AS so_n, + SUM(CASE WHEN c.is_single_offer = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.bids_received >= 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS so_vshare, + SUM(CASE WHEN c.is_high_markup = 1 THEN 1 ELSE 0 END) AS hm_k, + SUM(CASE WHEN c.is_high_markup IS NOT NULL THEN 1 ELSE 0 END) AS hm_n, + SUM(CASE WHEN c.is_high_markup = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.is_high_markup IS NOT NULL AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS hm_vshare + FROM contracts c + GROUP BY c.bidder_id +) AS agg +WHERE company_totals.bidder_id = agg.bidder_id; -- ── 3) authority_totals (per authority) ─────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS authority_totals ( authority_id TEXT PRIMARY KEY REFERENCES authorities(id), name TEXT NOT NULL, type_group TEXT, settlement TEXT, region TEXT, spent_eur REAL NOT NULL, contracts INTEGER NOT NULL, suppliers INTEGER NOT NULL, avg_eur REAL NOT NULL, primary_sector TEXT, - eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT + eu_eur REAL NOT NULL DEFAULT 0, first_date TEXT, last_date TEXT, + single_offer_k INTEGER, single_offer_n INTEGER, single_offer_value_share REAL, + high_markup_k INTEGER, high_markup_n INTEGER, high_markup_value_share REAL ); DELETE FROM authority_totals; INSERT INTO authority_totals (authority_id, name, type_group, settlement, region, spent_eur, contracts, suppliers, avg_eur, eu_eur, first_date, last_date) @@ -89,6 +132,24 @@ UPDATE authority_totals SET primary_sector = ( SELECT substr(t.cpv_code, 1, 2) FROM contracts c JOIN tenders t ON t.id = c.tender_id WHERE t.authority_id = authority_totals.authority_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1); +-- Per-subject risk aggregates — authority side (see the company_totals pass above for rationale). +UPDATE authority_totals SET + single_offer_k = agg.so_k, single_offer_n = agg.so_n, single_offer_value_share = agg.so_vshare, + high_markup_k = agg.hm_k, high_markup_n = agg.hm_n, high_markup_value_share = agg.hm_vshare +FROM ( + SELECT t.authority_id, + SUM(CASE WHEN c.is_single_offer = 1 THEN 1 ELSE 0 END) AS so_k, + SUM(CASE WHEN c.bids_received >= 1 THEN 1 ELSE 0 END) AS so_n, + SUM(CASE WHEN c.is_single_offer = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.bids_received >= 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS so_vshare, + SUM(CASE WHEN c.is_high_markup = 1 THEN 1 ELSE 0 END) AS hm_k, + SUM(CASE WHEN c.is_high_markup IS NOT NULL THEN 1 ELSE 0 END) AS hm_n, + SUM(CASE WHEN c.is_high_markup = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.is_high_markup IS NOT NULL AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS hm_vshare + FROM contracts c JOIN tenders t ON t.id = c.tender_id + GROUP BY t.authority_id +) AS agg +WHERE authority_totals.authority_id = agg.authority_id; -- home_totals uses the browsable leaderboard grains for authority/bidder counts, and the same -- freshness definition as refresh-slice.sql: latest in-corpus signed contract date. diff --git a/scripts/refresh-slice.sql b/scripts/refresh-slice.sql index e051a7ed2..540baab42 100644 --- a/scripts/refresh-slice.sql +++ b/scripts/refresh-slice.sql @@ -1259,6 +1259,17 @@ WHERE b.eik_normalized IN (SELECT eik FROM raw_ocds_parties WHERE eik IS NOT NUL WHERE bidder_key IS NOT NULL ); +-- Per-contract risk flags for the touched contracts, before the rollups aggregate them. Same +-- derivation as precompute.sql section 0b; runs after the recalc UPDATE above refreshed signing/current +-- EUR, so is_high_markup reads current figures. +UPDATE contracts SET + is_single_offer = CASE WHEN bids_received IS NOT NULL THEN (bids_received = 1) END, + is_high_markup = CASE WHEN value_flag = 'ok' + AND signing_value_eur IS NOT NULL AND current_value_eur IS NOT NULL + AND signing_value_eur > 0 + THEN ((current_value_eur - signing_value_eur) / signing_value_eur > 0.2) END +WHERE id IN (SELECT id FROM refresh_touched_contracts); + -- 6) Refresh rollups + FTS. Only the D1-hot rollups are scoped to touched rows; cheaper rollups stay -- full-recomputed in isolated batches so convergence stays simple. -- @refresh-batch company-totals @@ -1275,6 +1286,25 @@ UPDATE company_totals SET primary_sector = ( WHERE c.bidder_id = company_totals.bidder_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1) WHERE bidder_id IN (SELECT bidder_id FROM refresh_touched_bidders); +-- Per-subject risk aggregates for the touched bidders (see precompute.sql for rationale). +UPDATE company_totals SET + single_offer_k = agg.so_k, single_offer_n = agg.so_n, single_offer_value_share = agg.so_vshare, + high_markup_k = agg.hm_k, high_markup_n = agg.hm_n, high_markup_value_share = agg.hm_vshare +FROM ( + SELECT c.bidder_id, + SUM(CASE WHEN c.is_single_offer = 1 THEN 1 ELSE 0 END) AS so_k, + SUM(CASE WHEN c.bids_received >= 1 THEN 1 ELSE 0 END) AS so_n, + SUM(CASE WHEN c.is_single_offer = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.bids_received >= 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS so_vshare, + SUM(CASE WHEN c.is_high_markup = 1 THEN 1 ELSE 0 END) AS hm_k, + SUM(CASE WHEN c.is_high_markup IS NOT NULL THEN 1 ELSE 0 END) AS hm_n, + SUM(CASE WHEN c.is_high_markup = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.is_high_markup IS NOT NULL AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS hm_vshare + FROM contracts c + WHERE c.bidder_id IN (SELECT bidder_id FROM refresh_touched_bidders) + GROUP BY c.bidder_id +) AS agg +WHERE company_totals.bidder_id = agg.bidder_id; -- @refresh-batch authority-totals DELETE FROM authority_totals WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); @@ -1290,6 +1320,25 @@ UPDATE authority_totals SET primary_sector = ( WHERE t.authority_id = authority_totals.authority_id AND c.amount_eur IS NOT NULL AND COALESCE(t.cpv_code,'') <> '' GROUP BY substr(t.cpv_code, 1, 2) ORDER BY SUM(c.amount_eur) DESC, substr(t.cpv_code, 1, 2) LIMIT 1) WHERE authority_id IN (SELECT authority_id FROM refresh_touched_authorities); +-- Per-subject risk aggregates for the touched authorities (see precompute.sql for rationale). +UPDATE authority_totals SET + single_offer_k = agg.so_k, single_offer_n = agg.so_n, single_offer_value_share = agg.so_vshare, + high_markup_k = agg.hm_k, high_markup_n = agg.hm_n, high_markup_value_share = agg.hm_vshare +FROM ( + SELECT t.authority_id, + SUM(CASE WHEN c.is_single_offer = 1 THEN 1 ELSE 0 END) AS so_k, + SUM(CASE WHEN c.bids_received >= 1 THEN 1 ELSE 0 END) AS so_n, + SUM(CASE WHEN c.is_single_offer = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.bids_received >= 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS so_vshare, + SUM(CASE WHEN c.is_high_markup = 1 THEN 1 ELSE 0 END) AS hm_k, + SUM(CASE WHEN c.is_high_markup IS NOT NULL THEN 1 ELSE 0 END) AS hm_n, + SUM(CASE WHEN c.is_high_markup = 1 AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END) + / NULLIF(SUM(CASE WHEN c.is_high_markup IS NOT NULL AND c.amount_eur > 0 THEN c.amount_eur ELSE 0 END), 0) AS hm_vshare + FROM contracts c JOIN tenders t ON t.id = c.tender_id + WHERE t.authority_id IN (SELECT authority_id FROM refresh_touched_authorities) + GROUP BY t.authority_id +) AS agg +WHERE authority_totals.authority_id = agg.authority_id; -- @refresh-batch flow-pairs DELETE FROM flow_pairs;